(四)Kafka 再均衡监听器 实战小例子

项目介绍

本示例中，生产者发送50条消息给消费者为3的群组。在消费者群组中，第三个线程会中途退出群组，借此，我们可以观察分区再均衡现象。

依赖

<dependency>
    <groupId>org.apache.kafka</groupId>
    <artifactId>kafka-clients</artifactId>
    <version>2.3.0</version>
</dependency>

生产者

package Rebalance;

import org.apache.kafka.clients.producer.*;
import org.apache.kafka.common.serialization.StringSerializer;

import java.util.Properties;
import java.util.concurrent.CountDownLatch;
import java.util.concurrent.ExecutorService;
import java.util.concurrent.Executors;

/**
 * @Author Natasha
 * @Description
 * @Date 2020/11/3 14:07
 **/
public class RebalanceProducer {
    private static final int MSG_SIZE = 50;
    private static ExecutorService executorService = Executors.newFixedThreadPool(Runtime.getRuntime().availableProcessors());
    private static CountDownLatch countDownLatch = new CountDownLatch(MSG_SIZE);

    public static void main(String[] args) {
        Properties properties = new Properties();
        properties.put(ProducerConfig.BOOTSTRAP_SERVERS_CONFIG,"120.27.233.226:9092");
        properties.put(ProducerConfig.KEY_SERIALIZER_CLASS_CONFIG, StringSerializer.class);
        properties.put(ProducerConfig.VALUE_SERIALIZER_CLASS_CONFIG, StringSerializer.class);
        KafkaProducer<String,String> producer = new KafkaProducer(properties);
        try {
            for(int i=0;i<MSG_SIZE;i++){
                ProducerRecord<String,String> record = new ProducerRecord("rebalance-topic-three-part","value" + i);
                executorService.submit(new ProduceWorker(record,producer,countDownLatch));
                Thread.sleep(1000);
            }
            countDownLatch.await();
        } catch (Exception e) {
            e.printStackTrace();
        } finally {
            producer.close();
            executorService.shutdown();
        }
    }
}

生产任务

package Rebalance;

import org.apache.kafka.clients.producer.Callback;
import org.apache.kafka.clients.producer.KafkaProducer;
import org.apache.kafka.clients.producer.ProducerRecord;
import org.apache.kafka.clients.producer.RecordMetadata;

import java.util.concurrent.CountDownLatch;

/**
 * @Author Natasha
 * @Description
 * @Date 2020/11/3 14:10
 **/

public class ProduceWorker implements Runnable{

    private ProducerRecord<String,String> record;
    private KafkaProducer<String,String> producer;
    private CountDownLatch countDownLatch;

    public ProduceWorker(ProducerRecord<String, String> record, KafkaProducer<String, String> producer, CountDownLatch countDownLatch) {
        this.record = record;
        this.producer = producer;
        this.countDownLatch = countDownLatch;
    }

    public void run() {
        final String id = "" + Thread.currentThread().getId();
        try {
            producer.send(record, new Callback() {
                public void onCompletion(RecordMetadata metadata, Exception exception) {
                    if(null != exception){
                        exception.printStackTrace();
                    }
                    if(null != metadata){
                        System.out.println(id+"|"+String.format("偏移量：%s,分区：%s", metadata.offset(),metadata.partition()));
                    }
                }
            });
            System.out.println(id+":数据["+record.key()+ "-" + record.value()+"]已发送。");
            countDownLatch.countDown();
        } catch (Exception e) {
            e.printStackTrace();
        }
    }
}

消费者

package Rebalance;

import org.apache.kafka.clients.consumer.ConsumerConfig;
import org.apache.kafka.common.serialization.StringDeserializer;

import java.util.Properties;
import java.util.concurrent.ExecutorService;
import java.util.concurrent.Executors;
/**
 * @Author Natasha
 * @Description
 * @Date 2020/11/3 14:37
 **/

public class RebalanceConsumer {

    public static final String GROUP_ID = "RebalanceConsumer";
    private static ExecutorService executorService = Executors.newFixedThreadPool(3);


    public static void main(String[] args) throws InterruptedException {
        Properties properties = new Properties();
        properties.put(ConsumerConfig.BOOTSTRAP_SERVERS_CONFIG,"120.27.233.226:9092");
        properties.put(ConsumerConfig.KEY_DESERIALIZER_CLASS_CONFIG, StringDeserializer.class);
        properties.put(ConsumerConfig.VALUE_DESERIALIZER_CLASS_CONFIG, StringDeserializer.class);
        properties.put(ConsumerConfig.GROUP_ID_CONFIG, RebalanceConsumer.GROUP_ID);
        properties.put(ConsumerConfig.ENABLE_AUTO_COMMIT_CONFIG, false);
        for(int i = 0; i < 2; i++){
            executorService.submit(new ConsumerWorker(false, properties));
        }
        Thread.sleep(5000);
        //用来被停止，观察保持运行的消费者情况
        new Thread(new ConsumerWorker(true, properties)).start();
    }
}

消费任务

package Rebalance;

import org.apache.kafka.clients.consumer.*;
import org.apache.kafka.common.TopicPartition;

import java.time.Duration;
import java.util.Collections;
import java.util.HashMap;
import java.util.Map;
import java.util.Properties;
/**
 * @Author Natasha
 * @Description
 * @Date 2020/11/3 14:13
 **/

public class ConsumerWorker  implements Runnable{

    private final KafkaConsumer<String,String> consumer;
    /*用来保存每个消费者当前读取分区的偏移量*/
    private final Map<TopicPartition, OffsetAndMetadata> currOffsets;
    private final boolean isStop;

    /*消息消费者配置*/
    public ConsumerWorker(boolean isStop, Properties properties) {
        this.isStop = isStop;
        this.consumer = new KafkaConsumer(properties);
        this.currOffsets = new HashMap();
        consumer.subscribe(Collections.singletonList("rebalance-topic-three-part"), new HandlerRebalance(currOffsets,consumer));
    }

    public void run() {
        final String id = "" + Thread.currentThread().getId();
        int count = 0;
        TopicPartition topicPartition;
        long offset;
        try {
            while(true){
                ConsumerRecords<String, String> records = consumer.poll(Duration.ofMillis(500));
                //业务处理
                //开始事务
                for(ConsumerRecord<String, String> record:records){
                    System.out.println(id+"|"+String.format( "处理主题：%s，分区：%d，偏移量：%d，" + "key：%s，value：%s", record.topic(),record.partition(), record.offset(),record.key(),record.value()));
                    topicPartition = new TopicPartition(record.topic(), record.partition());
                    offset = record.offset()+1;
                    currOffsets.put(topicPartition,new OffsetAndMetadata(offset, "no"));
                    count++;
                    //执行业务sql
                }
                if(currOffsets.size()>0){
                    for(TopicPartition topicPartitionkey:currOffsets.keySet()){
                        HandlerRebalance.partitionOffsetMap.put(topicPartitionkey, currOffsets.get(topicPartitionkey).offset());
                    }
                    //提交事务,同时将业务和偏移量入库
                }
                //如果stop参数为true,这个消费者消费到第5个时自动关闭
                if(isStop&&count>=5){
                    System.out.println(id+"-将关闭，当前偏移量为："+currOffsets);
                    consumer.commitSync();
                    break;
                }
                consumer.commitSync();
            }
        } finally {
            consumer.close();
        }
    }
}

再均衡监听器

package Rebalance;

import org.apache.kafka.clients.consumer.ConsumerRebalanceListener;
import org.apache.kafka.clients.consumer.KafkaConsumer;
import org.apache.kafka.clients.consumer.OffsetAndMetadata;
import org.apache.kafka.common.TopicPartition;

import java.util.Collection;
import java.util.Map;
import java.util.concurrent.ConcurrentHashMap;
/**
 * @Author Natasha
 * @Description
 * @Date 2020/11/3 14:14
 **/

public class HandlerRebalance implements ConsumerRebalanceListener {

    private final Map<TopicPartition, OffsetAndMetadata> currOffsets;
    private final KafkaConsumer<String,String> consumer;
    //private final Transaction  tr事务类的实例

    public HandlerRebalance(Map<TopicPartition, OffsetAndMetadata> currOffsets, KafkaConsumer<String, String> consumer) {
        this.currOffsets = currOffsets;
        this.consumer = consumer;
    }

    /*模拟一个保存分区偏移量的数据库表*/
    public final static ConcurrentHashMap<TopicPartition,Long> partitionOffsetMap = new ConcurrentHashMap();

    //分区再均衡之前
    public void onPartitionsRevoked(Collection<TopicPartition> partitions) {

        final String id = Thread.currentThread().getId()+"";
        System.out.println(id+"-onPartitionsRevoked参数值为："+partitions);
        System.out.println(id+"-服务器准备分区再均衡，提交偏移量。当前偏移量为：" + currOffsets);
        //开始事务
        //偏移量写入数据库
        System.out.println("分区偏移量表中：" + partitionOffsetMap);
        for(TopicPartition topicPartition:partitions){
            partitionOffsetMap.put(topicPartition, currOffsets.get(topicPartition).offset());
        }
        consumer.commitSync(currOffsets);
        //提交业务数和偏移量入库  tr.commit
    }

    //分区再均衡完成以后
    public void onPartitionsAssigned(Collection<TopicPartition> partitions) {
        final String id = "" + Thread.currentThread().getId();
        System.out.println(id+"-再均衡完成，onPartitionsAssigned参数值为："+partitions);
        System.out.println("分区偏移量表中："+partitionOffsetMap);
        for(TopicPartition topicPartition:partitions){
            System.out.println(id+"-topicPartition："+topicPartition);
            //模拟从数据库中取得上次的偏移量
            Long offset = partitionOffsetMap.get(topicPartition);
            if(offset==null)
                continue;
            //从特定偏移量处开始记录 (从指定分区中的指定偏移量开始消费)
            //这样就可以确保分区再均衡中的数据不错乱
            consumer.seek(topicPartition,partitionOffsetMap.get(topicPartition));
        }
    }
}

运行结果解析

先启动生产者RebalanceProducer后，可以看到生产数据如下：

14:数据[null-value0]已发送。
14|偏移量：26,分区：1
15:数据[null-value1]已发送。
15|偏移量：28,分区：0
16:数据[null-value2]已发送。
16|偏移量：28,分区：2
17:数据[null-value3]已发送。
17|偏移量：27,分区：1
18:数据[null-value4]已发送。
18|偏移量：29,分区：0
19:数据[null-value5]已发送。
19|偏移量：29,分区：2
20:数据[null-value6]已发送。
20|偏移量：28,分区：1
21:数据[null-value7]已发送。
21|偏移量：30,分区：0
22:数据[null-value8]已发送。
22|偏移量：30,分区：2
23:数据[null-value9]已发送。
23|偏移量：29,分区：1
24:数据[null-value10]已发送。
24|偏移量：31,分区：0

启动消费者RebalanceConsumer，首先可以看到初始分区再均衡：

13-onPartitionsRevoked参数值为：[]
14-onPartitionsRevoked参数值为：[]
13-服务器准备分区再均衡，提交偏移量。当前偏移量为：{}
14-服务器准备分区再均衡，提交偏移量。当前偏移量为：{}
分区偏移量表中：{}
分区偏移量表中：{}
17-onPartitionsRevoked参数值为：[]
17-服务器准备分区再均衡，提交偏移量。当前偏移量为：{}
分区偏移量表中：{}

14-再均衡完成，onPartitionsAssigned参数值为：[rebalance-topic-three-part-1]
分区偏移量表中：{}
17-再均衡完成，onPartitionsAssigned参数值为：[rebalance-topic-three-part-2]
分区偏移量表中：{}
13-再均衡完成，onPartitionsAssigned参数值为：[rebalance-topic-three-part-0]
分区偏移量表中：{}
14-topicPartitionrebalance-topic-three-part-1
13-topicPartitionrebalance-topic-three-part-0
17-topicPartitionrebalance-topic-three-part-2

随着程序的继续执行启，来到第三线程关闭之前：

13|处理主题：rebalance-topic-three-part，分区：0，偏移量：51，key：null，value：value0
17|处理主题：rebalance-topic-three-part，分区：2，偏移量：51，key：null，value：value1
14|处理主题：rebalance-topic-three-part，分区：1，偏移量：49，key：null，value：value2
13|处理主题：rebalance-topic-three-part，分区：0，偏移量：52，key：null，value：value3
17|处理主题：rebalance-topic-three-part，分区：2，偏移量：52，key：null，value：value4
14|处理主题：rebalance-topic-three-part，分区：1，偏移量：50，key：null，value：value5
13|处理主题：rebalance-topic-three-part，分区：0，偏移量：53，key：null，value：value6
17|处理主题：rebalance-topic-three-part，分区：2，偏移量：53，key：null，value：value7
14|处理主题：rebalance-topic-three-part，分区：1，偏移量：51，key：null，value：value8
13|处理主题：rebalance-topic-three-part，分区：0，偏移量：54，key：null，value：value9
17|处理主题：rebalance-topic-three-part，分区：2，偏移量：57，key：null，value：value10
14|处理主题：rebalance-topic-three-part，分区：1，偏移量：52，key：null，value：value11
13|处理主题：rebalance-topic-three-part，分区：0，偏移量：55，key：null，value：value12
17|处理主题：rebalance-topic-three-part，分区：2，偏移量：55，key：null，value：value13

//注意这里，上面17处理的偏移量是55，处理完后，偏移量到了56，如下
17-将关闭，当前偏移量为：{rebalance-topic-three-part-2=OffsetAndMetadata{offset=56, leaderEpoch=null, metadata='no'}}

14|处理主题：rebalance-topic-three-part，分区：1，偏移量：53，key：null，value：value14
13|处理主题：rebalance-topic-three-part，分区：0，偏移量：54，key：null，value：value15
14|处理主题：rebalance-topic-three-part，分区：1，偏移量：55，key：null，value：value17
13|处理主题：rebalance-topic-three-part，分区：0，偏移量：58，key：null，value：value18

第三线程关闭后，分区再平衡：

13-onPartitionsRevoked参数值为：[rebalance-topic-three-part-0]
14-onPartitionsRevoked参数值为：[rebalance-topic-three-part-1]

13-服务器准备分区再均衡，提交偏移量。当前偏移量为：{rebalance-topic-three-part-0=OffsetAndMetadata{offset=58, leaderEpoch=null, metadata='no'}}
分区偏移量表中：{rebalance-topic-three-part-2=56, rebalance-topic-three-part-1=55, rebalance-topic-three-part-0=58}
14-服务器准备分区再均衡，提交偏移量。当前偏移量为：{rebalance-topic-three-part-1=OffsetAndMetadata{offset=55, leaderEpoch=null, metadata='no'}}
分区偏移量表中：{rebalance-topic-three-part-2=56, rebalance-topic-three-part-1=55, rebalance-topic-three-part-0=58}

14-再均衡完成，onPartitionsAssigned参数值为：[rebalance-topic-three-part-2]
13-再均衡完成，onPartitionsAssigned参数值为：[rebalance-topic-three-part-1, rebalance-topic-three-part-0]
分区偏移量表中：{rebalance-topic-three-part-2=56, rebalance-topic-three-part-1=55, rebalance-topic-three-part-0=58}
13-topicPartitionrebalance-topic-three-part-1
分区偏移量表中：{rebalance-topic-three-part-2=56, rebalance-topic-three-part-1=55, rebalance-topic-three-part-0=58}
14-topicPartitionrebalance-topic-three-part-2
13-topicPartitionrebalance-topic-three-part-0