kafka 消费消息实战

简述：kafka 获取消息，添加队列queue,再从队列中通过工厂+策略模式消费消息

kafkaConsume:

@Slf4j
@Component
public class KafkaConsume {

   
    @KafkaListener(groupId = "${kafka.collection}", topics = "erp_invoice")
    public void method(ConsumerRecord<String, Object> record, @Header(KafkaHeaders.RECEIVED_TOPIC) String topic, Consumer consumer) {
        log.warn("消费者收到消息:{}  topic:{}", record.value(), topic);
        /*
         * 如果需要手工提交异步 consumer.commitSync();
         * 手工同步提交 consumer.commitAsync()
         */
        try {
            Optional<Object> msg = Optional.ofNullable(record.value());
            if (msg.isPresent()) {
                Object realyMsg = msg.get();
                //加入队列
                if (realyMsg!=null) {
                    pullLogQueue(ConsumeQueueTypeEnum.ERP.getCode(), realyMsg);
                }
            }
        } catch (Exception e) {
            log.error("[kafka-消费异常] doConsumer Error {} ", ExceptionUtils.getFullStackTrace(e));
            //添加队列失败，消费异常添加到消费失败表
            ...
        } finally {
            consumer.commitSync();
        }
    }



    @KafkaListener(groupId = "${kafka.collection}", topics = "${kafka.wlzgTopic}")
    public void wlzgIncr(ConsumerRecord<String, Object> record, @Header(KafkaHeaders.RECEIVED_TOPIC) String topic, Consumer consumer) {
        log.warn("消费者收到消息:{}  topic:{}", record.value(), topic);
        /*
         * 如果需要手工提交异步 consumer.commitSync();
         * 手工同步提交 consumer.commitAsync()
         */
        try {
            Optional<Object> msg = Optional.ofNullable(record.value());
            if (msg.isPresent()) {
                Object realyMsg = msg.get();
                //加入队列
                if (realyMsg!=null) {
                    pullLogQueue(ConsumeQueueTypeEnum.WLZG.getCode(), realyMsg);
                }
            }
        } catch (Exception e) {
            log.error("[kafka-消费异常] doConsumer Error {} ", ExceptionUtils.getFullStackTrace(e));
            //添加到消费失败表中
              ...
        } finally {
            consumer.commitSync();
        }
    }


    //加入队列
    private void pullLogQueue(String type, Object realyMsg) {
        //加入队列
        KafkaQueueEntity entity = new KafkaQueueEntity();
        entity.setMsgType(type);
        entity.setRealyMsg(realyMsg);

        KafkaQueue queue = KafkaQueue.getInstance();
        if (queue.isFull()){
            queue.handleDiscardedLog();
        }else {
            if (!queue.push(entity)){
                queue.handleAddFailLog(entity);
            };
        }
    }

kafkaQueue

@Log4j
public class KafkaQueue {
    //队列大小
    public static final int QUEUE_MAX_SIZE = 1024;

    public static final int QUEUE_REMOVE_SIZE = 500;

    private static KafkaQueue logQueue = new KafkaQueue();

    /**
     * 阻塞队列
     */
    private BlockingQueue<KafkaQueueEntity> logBlockingQueue = new LinkedBlockingQueue<>(QUEUE_MAX_SIZE);

   
    public KafkaQueue() {
    }

    /**
     * 获得单例
     *
     * @return
     */
    public static KafkaQueue getInstance() {
        return logQueue;
    }


    /**
     * 入队
     *
     * @param kafkaQueueEntity
     * @return
     */
    public boolean push(KafkaQueueEntity kafkaQueueEntity) {
        return this.logBlockingQueue.offer(kafkaQueueEntity);
    }

    /**
     * 出队
     *
     * @return
     */
    public KafkaQueueEntity pop() {
        KafkaQueueEntity take = null;
        try {
            take = this.logBlockingQueue.take();
        } catch (InterruptedException ex) {
            throw new RRException("队列出队异常!", ex);
        }
        return take;
    }

    /**
     * 返回队列元素个数
     *
     * @return
     */
    public int size() {
        return this.logBlockingQueue.size();
    }

    public synchronized boolean isFull() {
        return logBlockingQueue.size() == QUEUE_MAX_SIZE;
    }


    /**
     * 队列满了 处理被丢弃的日志事件
     */
    @Autowired
    private KafkaConsumeFailLogService kafkaCustomerFailLogService;
    public void handleDiscardedLog() {
        List<KafkaQueueEntity> removeList=Lists.newCopyOnWriteArrayList();
        //随机取出500条数据
        int i = logBlockingQueue.drainTo(removeList, QUEUE_REMOVE_SIZE);
        //存入消费失败表中或根据实际场景处理
         ...
    }

    /**
     * 插入队列失败 处理日志事件
     */
    public void handleAddFailLog(Object realyMsg) {
        log.info("队列插入失败,丢弃的日志报错信息：" + realyMsg);
    }


}

kafka消费消息

@Slf4j
@Component
public class ConsumeKafkaQueue {

    @Autowired
    private KafkaConsumeFactory kafkaConsumeFactory;

    @PostConstruct
    public void startLogThread() {
        ThreadPoolExecutor threadPoolExecutor = new ThreadPoolExecutor(100, 300, 200, TimeUnit.SECONDS, new LinkedBlockingQueue<>(16));
        threadPoolExecutor.submit(new PopSysLog(kafkaConsumeFactory));
    }


    class PopSysLog implements Runnable {

        private KafkaConsumeFactory kafkaConsumeFactory;

        public PopSysLog(KafkaConsumeFactory kafkaConsumeFactory) {
            this.kafkaConsumeFactory = kafkaConsumeFactory;
        }

        @Override
        public void run() {
            KafkaQueue queue = KafkaQueue.getInstance();
            while (true) {
                try {
                   //出队
                    KafkaQueueEntity queueEntity = queue.pop();
                    if (queueEntity != null) {
                        //调用工厂方法
                        Boolean aBoolean = kafkaConsumeFactory.getKafkaConsumeQueue(ConsumeQueueTypeEnum.getStatusValue(queueEntity.getMsgType())).consumeMsgByType(queueEntity.getRealyMsg());
                    }
                } catch (Exception e) {
                    log.error("处理消费信息异常",e);
                }
            }
        }
    }

}

工厂+策略模式处理业务需求

public interface KafkaConsumeQueue {

    //根据指定类型消费消息
    public abstract Boolean consumeMsgByType(Object realyMsg);

}

工厂

@Component
public class KafkaConsumeFactory {

    @Autowired
    Map<String, KafkaConsumeQueue> map = new ConcurrentHashMap<>();

    public KafkaConsumeQueue getKafkaConsumeQueue(ConsumeQueueTypeEnum consumeQueueTypeEnum) {

        KafkaConsumeQueue consumeQueue = map.get(consumeQueueTypeEnum.getValue());
        if (consumeQueue == null) {
            throw new RRException("未定义的消费类型");
        }
        return consumeQueue;
    }
}

处理实际业务

@Component("WL")
@Slf4j
public class ConsumeWLInfo implements KafkaConsumeQueue {

    @Autowired
    private KafkaConsumeFailLogService kafkaConsumeFailLogService;

    @Override
    public Boolean consumeMsgByType(Object realyMsg) {

        try {
            JSONObject jsonObject = JSON.parseObject(realyMsg.toString());
            //jsonArray转map
            if (Objects.nonNull(jsonObject)) {
                String version = (String) jsonObject.get("bdmp_version");
               ...
            }
            return Boolean.TRUE;
        } catch (Exception e) {
            //添加到消费失败表中
          ...
        }
    }

}

枚举 value 需要和 ConsumeWLInfo 中@Component("WL") 相同

public enum ConsumeQueueTypeEnum {

    TX("1","TX"),//通信

    WL("2","WL");//物流

    private String code;

    private String value;

    ConsumeQueueTypeEnum(String code, String value) {
        this.code = code;
        this.value = value;
    }

    public String getCode() {
        return code;
    }

    public String getValue() {
        return value;
    }

}