Pulsar with Spring | Code Samples

依赖

<dependency>
    <groupId>org.apache.pulsar</groupId>
    <artifactId>pulsar-client</artifactId>
    <version>2.10.0</version>
</dependency>

<!-- 可选 -->
<dependency>
    <groupId>dev.failsafe</groupId>
    <artifactId>failsafe</artifactId>
    <version>3.2.4</version>
</dependency>

Spring 集成使用

pulsar.url=${PULSAR_URL}
pulsar.topic=${PULSAR_TOPIC}
pulsar.token=${PULSAR_TOKEN}

PulsarConfig Class

一个 Pulsar 配置类。事实上，你也可以在任何 SpringBean 的构造方法中注入 PulsarClient，再构建 Producer 或 Consumer，SpringBean 实现 DisposableBean 来控制退出程序时调用 Producer/Consumer 的 close 方法。

@Configuration
public class PulsarConfig {
    
    @Value("${pulsar.url}")
    private String url;

    @Value("${pulsar.topic}")
    private String topic;

    @Value("${pulsar.token}")
    private String token;

    @Bean(destroyMethod = "close")
    public PulsarClient pulsarClient() throws PulsarClientException {
        return PulsarClient.builder()
            .serviceUrl(url)
            .authentication(AuthenticationFactory.token(token))
            .build();        
    }

    @Bean(destroyMethod = "close")
    public Producer<String> producer() throws PulsarClientException {
        return pulsarClient().newProducer(Schema.STRING)
            .topic(topic)
            .create();
    }

    @Bean(destroyMethod = "close")
    public Consumer<String> consumer() throws PulsarClientException {
        return pulsarClient().newConsumer(Schema.STRING)
            .topic(topic)
            .subscriptionType(SubscriptionType.Shared)
            .subscriptionName("subscriptionName")
            .subscribe();
    }
    
}

额外的，注册一个 Spring 的执行器，当然你也可以用 juc 的，但记得要把里边的线程都设置未守护线程

    @Bean
    public ThreadPoolTaskExecutor threadPoolTaskExecutor() {
        ThreadPoolTaskExecutor executor = new ThreadPoolTaskExecutor();
        executor.setCorePoolSize(3);
        executor.setMaxPoolSize(10);
        executor.setDaemon(true);
        executor.setThreadNamePrefix("pulsar-handler-pool");
        return executor;
    }

EventHandleService

一个事件处理服务 EventHandleService，来注册和执行 pulsar message 处理器

@Slf4j
@Service
public class EventHandleService implements DisposableBean{
    
    private final Consumer<String> consumer;

    private final Set<java.util.function.Consumer<Message<String>>> handlers = new HashSet<>();

    private final TaskExecutor work;

    private final AtomicBoolean start = new AtomicBoolean(false);

    public EventHandleService(Consumer<String> consumer, @Qualifier("threadPoolTaskExecutor") TaskExecutor executor) {
        this.consumer = consumer;
        this.work = executor;
        start.set(true);
        this.work.execute(this::schedule);
    
    }

    public void registerHandler(java.util.function.Consumer<Message<String>> handler) {
        handlers.add(handler);
    }

    public void removeHandler(java.util.function.Consumer<Message<String>> handler) {
        handlers.remove(handler);
    }

    private void schedule() {
        while (start.get()) {
            try {
                Message<String> message = consumer.receive();
                if (handlers.isEmpty()) {
                    consumer.reconsumeLater(message, 5, TimeUnit.SECONDS);
                    continue;
                }
                for (java.util.function.Consumer<Message<String>> handler : handlers) {
                    work.execute( () -> execute(handler, message) );
                }
            } catch (PulsarClientException e) {
                log.warn("Receive pulsar message failed", e);
            }
        }
    }

    private void execute(java.util.function.Consumer<Message<String>> handler, Message<String> message) {
        try {
            handler.accept(message);
            consumer.acknowledge(message);
        } catch (Exception e) {
            consumer.negativeAcknowledge(message);
            log.warn("Handle pulsar message failed", e);
        }
    }

    @Override
    public void destroy() throws Exception {
        start.set(false);        
    }

}

支持 Timeout

可选的，可以设置事件处理器的最大可执行时间

    private final Map<java.util.function.Consumer<Message<String>>, Timeout<Object>> timeoutMap = new HashMap<>();

    public void registerHandler(java.util.function.Consumer<Message<String>> handler, long timeout) {
        timeoutMap.put(handler, Timeout.of(Duration.ofMillis(timeout)));
        handlers.add(handler);
    }

    private void execute(java.util.function.Consumer<Message<String>> handler, Message<String> message) {
        try {
            Timeout<Object> timeout = timeoutMap.get(handler);
            if (timeout == null) {
                handler.accept(message);
            } else {
                Failsafe.with(timeout).run(() -> handler.accept(message));
            }
            consumer.acknowledge(message);
        } catch (Exception e) {
            consumer.negativeAcknowledge(message);
            log.warn("Handle pulsar message failed", e);
        }
    }
    

示例

Producer 和 EventHandleService 可在任何服务类中使用，一个示例

@Slf4j
@Service
public class TimeService implements InitializingBean {

    @Autowired
    private Producer<String> producer;

    @Autowired
    private EventHandleService eventHandleService;

    private final AtomicReference<LocalDateTime> lastDateTime = new AtomicReference<>(LocalDateTime.now());
    
    public static final String UPDATE_LAST_DATETIME_EVENT = "event:TimeService:updateLastDateTime";

    public LocalDateTime getLastDateTime() {
        return lastDateTime.get();
    }

    public LocalDateTime updateLastDateTime() {
        LocalDateTime now = LocalDateTime.now();
        lastDateTime.set(now);
        return now;
    }

    public void sendUpdateLastDateTimeEvent() {
        try {
            producer.send(UPDATE_LAST_DATETIME_EVENT);
        } catch (PulsarClientException e) {
            log.warn("Send pulsar message failed", e);
        }
    }

    private void handleUpdateLastDateTimeEvent(Message<String> event) {
        if (Objects.equals(event.getValue(), UPDATE_LAST_DATETIME_EVENT)) {
            updateLastDateTime();
            log.info("Handle event - {} {} {}", event.getProducerName(),  event.getMessageId(), event.getValue());
        }
    }

    @Override
    public void afterPropertiesSet() throws Exception {
        eventHandleService.registerHandler(this::handleUpdateLastDateTimeEvent);
    }
    
}

Java 原生使用

PulsarConsumerHelper

public class PulsarConsumerHelper <T> {

    private final Consumer<T> consumer;

    private java.util.function.Consumer<Message<T>> messageHandler;
    private java.util.function.Consumer<Throwable> exceptioHandler;

    private ExecutorService worker = Executors.newFixedThreadPool(2, new NamedThreadFactory("pulsar-consumer-helper-") );

    private final AtomicBoolean closed = new AtomicBoolean(false);

    /**
     * 创建 Helper, 暂定消息拉取直到设置消息处理器
     * @param consumer
     */
    public PulsarConsumerHelper(Consumer<T> consumer) {
        this.consumer = Objects.requireNonNull(consumer);
        this.consumer.pause();
        worker.submit(this::schedule);
    }

    /**
     * 设置消息处理器, 启动消息拉取
     * @param handler
     */
    public void handler(java.util.function.Consumer<Message<T>> handler) {
        this.messageHandler = handler;
        consumer.resume();
    }

    /**
     * 设置异常处理器
     * @param exceptioHandler
     */
    public void exceptionHandler(java.util.function.Consumer<Throwable> exceptioHandler) {
        this.exceptioHandler = exceptioHandler;
    }

    public void close() throws PulsarClientException {
        closed.set(true);
        worker.shutdown();
        consumer.close();
    }

    private void schedule() {
        while (!closed.get()) {
            try {
                Message<T> msg = consumer.receive();
                worker.submit(() -> run(msg));
            } catch (Exception e) {
                if (exceptioHandler != null) {
                    exceptioHandler.accept(e);
                }
            }
        }
    }

    private void run(Message<T> msg) {
        if (!closed.get()) {
            try {
                if (messageHandler != null) {
                    messageHandler.accept(msg);
                    consumer.acknowledge(msg);
                } else {
                    consumer.reconsumeLater(msg, 1, TimeUnit.SECONDS);
                }
            } catch (Exception e) {
                if (exceptioHandler != null) {
                    exceptioHandler.accept(e);
                }
                consumer.negativeAcknowledge(msg);
            }
        }
    }

}

• 为保证按消息顺序消费，需要把处理器线程数量设为 1 ( schedule 函数占用一个线程)
• 在 messageHandler 设置之前，应该暂停消费拉取 receive

支持 Timeout

    private Timeout<Object> timeout = Timeout.of(Duration.ofSeconds(2));


    /**
     * 设置消息处理超时时间
     * @param duration
     */
    public void setTimeout(long millis) {
        timeout = Timeout.of(Duration.ofMillis(millis));
    }
    
    private void run(Message<T> msg) {
        if (!closed.get()) {
            try {
                if (messageHandler != null) {
                    Failsafe.with(timeout).run(() -> messageHandler.accept(msg));
                    consumer.acknowledge(msg);
                } else {
                    consumer.reconsumeLater(msg, 1, TimeUnit.SECONDS);
                }
            } catch (TimeoutExceededException e ) {
                log.warn("Message [{}] handle timeout {}ms", msg.getMessageId(), e.getTimeout().getConfig().getTimeout().toMillis() );
            } catch (Exception e) {
                if (exceptioHandler != null) {
                    exceptioHandler.accept(e);
                }
                consumer.negativeAcknowledge(msg);
            }
        }
    }

示例

        PulsarClient client = PulsarClient.builder()
            .serviceUrl("pulsar://localhost:6650")
            .build();

        Producer<String> producer = client.newProducer(Schema.STRING)
            .autoUpdatePartitions(false)
            .producerName("vertx-play-producer")
            .topic("test-topic")
            .create();

        producer.send("Hello from vertx 1");
        producer.send("Hello from vertx 2");
        producer.send("Hello from vertx 3");
        producer.send("Hello from vertx 4");
        producer.send("Hello from vertx 5");
        producer.send("Hello from vertx 6");
        producer.send("Hello from vertx 7");

        Consumer<String> consumer = client.newConsumer(Schema.STRING)
            .topic("test-topic")
            .subscriptionName("vertx-play-subscription")
            .subscribe();

        PulsarConsumerHelper<String> helper = new PulsarConsumerHelper<>(consumer);

        helper.handler(msg -> {
            log.info("[{}] {}", msg.getMessageId(), msg.getValue());
        });
        
        helper.exceptionHandler(t -> {
            log.warn("Handle pulsar message failed", t);
        });

可见在 worker 为 1 情况下，消息有序处理。

如果将 worker 设为 4 ，消息并行处理将不保证有序。

    private ExecutorService worker = Executors.newFixedThreadPool(5, new NamedThreadFactory("pulsar-consumer-helper-") );

后记

多线程处理拉取的消息时，是不保证消息顺序执行的；如果需要消息顺序执行，请使用单线程。

reconsumeLater 和 negativeAcknowledge 区别

Pulsar Message 在调用reconsumeLater 和 negativeAcknowledge之后都可以在此消费，它们的区别如下

-	Topic	MessageId	延时	*enableRetry
reconsumeLater	重试Topic	重试Topic中 MessageId	支持，见重载方法	true
negativeAcknowledge	原Topic	原 MessageId	支持，见negativeAckRedeliveryDelay::negativeAckRedeliveryDelay	false