消费模式
在使用Rocketmq时,无论是发送消息还是消费消息,都需要指定Topic,这让我们误以为Rocketmq管理消息的单位是Topic,其实不是的,一个Topic下面有多个Queue,每一条消息会进入其中一个Queue,那作为消费者,要如何消费Queue中的消息呢?
在Rocketmq中,消费者有两种消费模式,广播模式和集群模式,默认情况下使用的是集群模式
广播模式
广播模式是指消费组中的每一个消费者都能收到所有消息队列中的消息
集群模式
集群模式是指消息队列中的每一条消息只能被消费组中的其中一个消费者消费,这样就涉及到消息队列的分配问题(哪个消费者消费哪个消息队列),Rocketmq中提供了非常多的分配方式,如下图,便是两个消费者各自消费两个broker中两条队列
那么问题来了,如果消费者组中的消费者数量比消息队列多,那怎么办?
如下图,狼多肉少,自然会有狼吃不到肉,这种情况下,会有闲置的消费者,所以如果线上出现broker消息积压,无限增加消费者是有瓶颈的,我们除了增加消费者节点,还应该排查发送端或者broker是否有问题
Rocketmq是如何实现Queue的分配的呢?
因为消费者组中的消费者是不断变化的,Rocketmq使用了一个定时任务来不断地进行Queue的重分配
在消费者启动时,会启动下面两个至关重要的服务
public void start() throws MQClientException {
synchronized (this) {
switch (this.serviceState) {
case CREATE_JUST:
......
// 拉取消息服务
this.pullMessageService.start();
// 消费者重平衡服务
this.rebalanceService.start();
......
default:
break;
}
}
}
我们先看看消费者重平衡服务做了什么
// 循环遍历每一个topic
private void rebalanceByTopic(final String topic, final boolean isOrder) {
switch (messageModel) {
// 广播模式
case BROADCASTING: {
// 获取到与该Topic有关的所有队列
Set<MessageQueue> mqSet = this.topicSubscribeInfoTable.get(topic);
if (mqSet != null) {
// 更新ProcessQueueTable表
boolean changed = this.updateProcessQueueTableInRebalance(topic, mqSet, isOrder);
if (changed) {
this.messageQueueChanged(topic, mqSet, mqSet);
log.info("messageQueueChanged {} {} {} {}",
consumerGroup,
topic,
mqSet,
mqSet);
}
} else {
log.warn("doRebalance, {}, but the topic[{}] not exist.", consumerGroup, topic);
}
break;
}
// 集群模式
case CLUSTERING: {
// 获取到与该Topic有关的所有队列
Set<MessageQueue> mqSet = this.topicSubscribeInfoTable.get(topic);
// 获取到消费组中的所有消费者id
List<String> cidAll = this.mQClientFactory.findConsumerIdList(topic, consumerGroup);
if (null == mqSet) {
if (!topic.startsWith(MixAll.RETRY_GROUP_TOPIC_PREFIX)) {
log.warn("doRebalance, {}, but the topic[{}] not exist.", consumerGroup, topic);
}
}
if (null == cidAll) {
log.warn("doRebalance, {} {}, get consumer id list failed", consumerGroup, topic);
}
if (mqSet != null && cidAll != null) {
List<MessageQueue> mqAll = new ArrayList<MessageQueue>();
mqAll.addAll(mqSet);
// 对消息队列跟消费者id进行排序,因为消费者客户端可能有多个,要保证所有的客户端根据分配策略算出来的结果是一致的,
// 所以要先进行排序
Collections.sort(mqAll);
Collections.sort(cidAll);
AllocateMessageQueueStrategy strategy = this.allocateMessageQueueStrategy;
List<MessageQueue> allocateResult = null;
try {
// 根据分配策略进行分配,获取分配结果
allocateResult = strategy.allocate(
this.consumerGroup,
this.mQClientFactory.getClientId(),
mqAll,
cidAll);
} catch (Throwable e) {
log.error("AllocateMessageQueueStrategy.allocate Exception. allocateMessageQueueStrategyName={}", strategy.getName(),
e);
return;
}
Set<MessageQueue> allocateResultSet = new HashSet<MessageQueue>();
if (allocateResult != null) {
allocateResultSet.addAll(allocateResult);
}
// 更新ProcessQueueTable表
boolean changed = this.updateProcessQueueTableInRebalance(topic, allocateResultSet, isOrder);
if (changed) {
log.info(
"rebalanced result changed. allocateMessageQueueStrategyName={}, group={}, topic={}, clientId={}, mqAllSize={}, cidAllSize={}, rebalanceResultSize={}, rebalanceResultSet={}",
strategy.getName(), consumerGroup, topic, this.mQClientFactory.getClientId(), mqSet.size(), cidAll.size(),
allocateResultSet.size(), allocateResultSet);
this.messageQueueChanged(topic, mqSet, allocateResultSet);
}
}
break;
}
default:
break;
}
}
更新ProcessQueueTable表的源码如下
private boolean updateProcessQueueTableInRebalance(final String topic, final Set<MessageQueue> mqSet,
final boolean isOrder) {
boolean changed = false;
// 获取旧的processQueue
Iterator<Entry<MessageQueue, ProcessQueue>> it = this.processQueueTable.entrySet().iterator();
// 根据新分配的消息队列更新之前的processQueue
// 遍历旧的processQueue
while (it.hasNext()) {
Entry<MessageQueue, ProcessQueue> next = it.next();
MessageQueue mq = next.getKey();
ProcessQueue pq = next.getValue();
if (mq.getTopic().equals(topic)) {
// 如果新分配的消息队列集合中没有这个旧消息队列
if (!mqSet.contains(mq)) {
// 移除消息队列
pq.setDropped(true);
// 解锁,下面会讲述为什么要解锁
if (this.removeUnnecessaryMessageQueue(mq, pq)) {
it.remove();
changed = true;
log.info("doRebalance, {}, remove unnecessary mq, {}", consumerGroup, mq);
}
// 如果消息队列过期,进行移除
// 过期的判断是(System.currentTimeMillis() - this.lastPullTimestamp) > PULL_MAX_IDLE_TIME;
// lastPullTimestamp是上一次拉取成功的时间,PULL_MAX_IDLE_TIME默认是120000ms
} else if (pq.isPullExpired()) {
switch (this.consumeType()) {
case CONSUME_ACTIVELY:
break;
case CONSUME_PASSIVELY:
pq.setDropped(true);
if (this.removeUnnecessaryMessageQueue(mq, pq)) {
it.remove();
changed = true;
log.error("[BUG]doRebalance, {}, remove unnecessary mq, {}, because pull is pause, so try to fixed it",
consumerGroup, mq);
}
break;
default:
break;
}
}
}
}
List<PullRequest> pullRequestList = new ArrayList<PullRequest>();
for (MessageQueue mq : mqSet) {
// 如果是新增加的消息队列
if (!this.processQueueTable.containsKey(mq)) {
if (isOrder && !this.lock(mq)) {
log.warn("doRebalance, {}, add a new mq failed, {}, because lock failed", consumerGroup, mq);
continue;
}
this.removeDirtyOffset(mq);
// 添加一个新的ProcessQueue
ProcessQueue pq = new ProcessQueue();
long nextOffset = -1L;
try {
nextOffset = this.computePullFromWhereWithException(mq);
} catch (Exception e) {
log.info("doRebalance, {}, compute offset failed, {}", consumerGroup, mq);
continue;
}
if (nextOffset >= 0) {
ProcessQueue pre = this.processQueueTable.putIfAbsent(mq, pq);
if (pre != null) {
log.info("doRebalance, {}, mq already exists, {}", consumerGroup, mq);
} else {
log.info("doRebalance, {}, add a new mq, {}", consumerGroup, mq);
// 新增一个拉取消息的请求
PullRequest pullRequest = new PullRequest();
pullRequest.setConsumerGroup(consumerGroup);
pullRequest.setNextOffset(nextOffset);
pullRequest.setMessageQueue(mq);
pullRequest.setProcessQueue(pq);
pullRequestList.add(pullRequest);
changed = true;
}
} else {
log.warn("doRebalance, {}, add new mq failed, {}", consumerGroup, mq);
}
}
}
this.dispatchPullRequest(pullRequestList);
return changed;
}
拉取消息
开头处我们讲了,消费者启动时会启动两个至关重要的服务,一个是消费者重平衡定时任务,另一个则是拉取消息的服务
同样的,我们直接看这个类的run方法
@Override
public void run() {
log.info(this.getServiceName() + " service started");
while (!this.isStopped()) {
try {
// PullRequest,对于这个对象会不会很熟悉,前面更新ProcessQueueTable表时,如果该消费者新增加了消费的消息队列,会新增一个
// PullRequest,此处会不断从阻塞队列中获取PullRequest,进行消息的拉取
PullRequest pullRequest = this.pullRequestQueue.take();
this.pullMessage(pullRequest);
} catch (InterruptedException ignored) {
} catch (Exception e) {
log.error("Pull Message Service Run Method exception", e);
}
}
log.info(this.getServiceName() + " service end");
}
我们先来看看PullRequest这个类的属性,这样可以帮助我们更容易看懂下面的源码
public class PullRequest {
// 消费者组
private String consumerGroup;
// 待拉取消息队列
private MessageQueue messageQueue;
// 消息处理队列
private ProcessQueue processQueue;
// 待拉取messageQueue偏移量
private long nextOffset;
// 是否被锁定
private boolean previouslyLocked = false;
}
整个pullMessage的方法比较长,我们只看比较关键的部分
public void pullMessage(final PullRequest pullRequest) {
// 此处会进行拉取消息的流控
long cachedMessageCount = processQueue.getMsgCount().get();
long cachedMessageSizeInMiB = processQueue.getMsgSize().get() / (1024 * 1024);
// 如果缓存消息的条数大于1000,会稍后再进行消息的拉取
if (cachedMessageCount > this.defaultMQPushConsumer.getPullThresholdForQueue()) {
this.executePullRequestLater(pullRequest, PULL_TIME_DELAY_MILLS_WHEN_FLOW_CONTROL);
if ((queueFlowControlTimes++ % 1000) == 0) {
log.warn(
"the cached message count exceeds the threshold {}, so do flow control, minOffset={}, maxOffset={}, count={}, size={} MiB, pullRequest={}, flowControlTimes={}",
this.defaultMQPushConsumer.getPullThresholdForQueue(), processQueue.getMsgTreeMap().firstKey(), processQueue.getMsgTreeMap().lastKey(), cachedMessageCount, cachedMessageSizeInMiB, pullRequest, queueFlowControlTimes);
}
return;
}
// 如果缓存消息的大小大于100M,会稍后再进行消息的拉取
if (cachedMessageSizeInMiB > this.defaultMQPushConsumer.getPullThresholdSizeForQueue()) {
this.executePullRequestLater(pullRequest, PULL_TIME_DELAY_MILLS_WHEN_FLOW_CONTROL);
if ((queueFlowControlTimes++ % 1000) == 0) {
log.warn(
"the cached message size exceeds the threshold {} MiB, so do flow control, minOffset={}, maxOffset={}, count={}, size={} MiB, pullRequest={}, flowControlTimes={}",
this.defaultMQPushConsumer.getPullThresholdSizeForQueue(), processQueue.getMsgTreeMap().firstKey(), processQueue.getMsgTreeMap().lastKey(), cachedMessageCount, cachedMessageSizeInMiB, pullRequest, queueFlowControlTimes);
}
return;
}
......
try {
// 这里会对服务端进行消息的拉取,pullCallback这个参数比较重要,拉取成功会对回调这个方法
this.pullAPIWrapper.pullKernelImpl(
pullRequest.getMessageQueue(),
subExpression,
subscriptionData.getExpressionType(),
subscriptionData.getSubVersion(),
pullRequest.getNextOffset(),
this.defaultMQPushConsumer.getPullBatchSize(),
sysFlag,
commitOffsetValue,
BROKER_SUSPEND_MAX_TIME_MILLIS,
CONSUMER_TIMEOUT_MILLIS_WHEN_SUSPEND,
CommunicationMode.ASYNC,
pullCallback
);
} catch (Exception e) {
log.error("pullKernelImpl exception", e);
this.executePullRequestLater(pullRequest, pullTimeDelayMillsWhenException);
}
}
我们看看这个回调方法干了什么
PullCallback pullCallback = new PullCallback() {
@Override
public void onSuccess(PullResult pullResult) {
if (pullResult != null) {
// 下文详细讲述了这个方法
pullResult = DefaultMQPushConsumerImpl.this.pullAPIWrapper.processPullResult(pullRequest.getMessageQueue(), pullResult,
subscriptionData);
switch (pullResult.getPullStatus()) {
// 如果拉取消息成功
case FOUND:
// 记录这次拉取消息是从哪个offset开始拉取的
long prevRequestOffset = pullRequest.getNextOffset();
// 记录下次该从哪个offset开始拉取消息
pullRequest.setNextOffset(pullResult.getNextBeginOffset());
long pullRT = System.currentTimeMillis() - beginTimestamp;
DefaultMQPushConsumerImpl.this.getConsumerStatsManager().incPullRT(pullRequest.getConsumerGroup(),
pullRequest.getMessageQueue().getTopic(), pullRT);
long firstMsgOffset = Long.MAX_VALUE;
// 如果拉去过来的消息为空
if (pullResult.getMsgFoundList() == null || pullResult.getMsgFoundList().isEmpty()) {
// 立刻执行下一次拉取
DefaultMQPushConsumerImpl.this.executePullRequestImmediately(pullRequest);
} else {
firstMsgOffset = pullResult.getMsgFoundList().get(0).getQueueOffset();
DefaultMQPushConsumerImpl.this.getConsumerStatsManager().incPullTPS(pullRequest.getConsumerGroup(),
pullRequest.getMessageQueue().getTopic(), pullResult.getMsgFoundList().size());
// 把消息放进processQueue中
boolean dispatchToConsume = processQueue.putMessage(pullResult.getMsgFoundList());
// 进行消费
DefaultMQPushConsumerImpl.this.consumeMessageService.submitConsumeRequest(
pullResult.getMsgFoundList(),
processQueue,
pullRequest.getMessageQueue(),
dispatchToConsume);
// 如果设置了拉取间隔时间的话
if (DefaultMQPushConsumerImpl.this.defaultMQPushConsumer.getPullInterval() > 0) {
// 进行等待,再进行拉取,这个方法是一个定时任务
DefaultMQPushConsumerImpl.this.executePullRequestLater(pullRequest,
DefaultMQPushConsumerImpl.this.defaultMQPushConsumer.getPullInterval());
} else {
// 没有设置间隔时间,立刻进行拉取
DefaultMQPushConsumerImpl.this.executePullRequestImmediately(pullRequest);
}
}
......
break;
......
}
}
}
@Override
public void onException(Throwable e) {
if (!pullRequest.getMessageQueue().getTopic().startsWith(MixAll.RETRY_GROUP_TOPIC_PREFIX)) {
log.warn("execute the pull request exception", e);
}
DefaultMQPushConsumerImpl.this.executePullRequestLater(pullRequest, pullTimeDelayMillsWhenException);
}
};
public PullResult processPullResult(final MessageQueue mq, final PullResult pullResult,
final SubscriptionData subscriptionData) {
PullResultExt pullResultExt = (PullResultExt) pullResult;
// 更新下次从哪个broke上面拉消息,broker端如果发现从这个broker拉取消息比较慢的话,会建议从主broker上拉取消息
this.updatePullFromWhichNode(mq, pullResultExt.getSuggestWhichBrokerId());
if (PullStatus.FOUND == pullResult.getPullStatus()) {
// 解码
ByteBuffer byteBuffer = ByteBuffer.wrap(pullResultExt.getMessageBinary());
List<MessageExt> msgList = MessageDecoder.decodes(byteBuffer);
// 如果设置了过滤条件,此处进行过滤
List<MessageExt> msgListFilterAgain = msgList;
if (!subscriptionData.getTagsSet().isEmpty() && !subscriptionData.isClassFilterMode()) {
msgListFilterAgain = new ArrayList<MessageExt>(msgList.size());
for (MessageExt msg : msgList) {
if (msg.getTags() != null) {
if (subscriptionData.getTagsSet().contains(msg.getTags())) {
msgListFilterAgain.add(msg);
}
}
}
}
// 如果有hook方法,执行hook方法
if (this.hasHook()) {
FilterMessageContext filterMessageContext = new FilterMessageContext();
filterMessageContext.setUnitMode(unitMode);
filterMessageContext.setMsgList(msgListFilterAgain);
this.executeHook(filterMessageContext);
}
for (MessageExt msg : msgListFilterAgain) {
String traFlag = msg.getProperty(MessageConst.PROPERTY_TRANSACTION_PREPARED);
if (Boolean.parseBoolean(traFlag)) {
msg.setTransactionId(msg.getProperty(MessageConst.PROPERTY_UNIQ_CLIENT_MESSAGE_ID_KEYIDX));
}
// 给每个msg上面添加一个最小offset和最大offset
MessageAccessor.putProperty(msg, MessageConst.PROPERTY_MIN_OFFSET,
Long.toString(pullResult.getMinOffset()));
MessageAccessor.putProperty(msg, MessageConst.PROPERTY_MAX_OFFSET,
Long.toString(pullResult.getMaxOffset()));
msg.setBrokerName(mq.getBrokerName());
}
pullResultExt.setMsgFoundList(msgListFilterAgain);
}
pullResultExt.setMessageBinary(null);
return pullResult;
}
消费消息
Rocketmq消费消息有两种方式,一种是并发消费,一种是顺序消费,并发消费是不考虑顺序性的,从服务端拿到消息就会直接进行消费,顺序性则会进行更多考虑,但是需要注意的是,这里的顺序性的单位是Queue,不是Topic,假如顺序发送的消息进入到不同的Queue,消费者都有可能是不同的,在这种情况下,是无法保证顺序性的,Rocketmq只能保证单个Queue是顺序消费的
ConsumeMessageOrderlyService:顺序消费
ConsumeMessageConcurrentlyService:并发消费
在看顺序消费是怎么做之前,我们先思考一下,如果自己来做顺序消费要怎么做?
这是一个比较常规的问题,有序性问题一般都是用加锁解决,但是在Rocketmq中,这个问题还要更复杂一些,因为Rocketmq无时无刻不在发生着重平衡,某个Queue这分钟属于这个消费者,下分钟可能属于下个消费者,在转移消费者的时候,顺序性就很难保证了,所以Rocketmq用了两把锁保证顺序性,一个在消费者端,一把在broker端,Rocketmq要保证同一时刻只有一个消费者在消费某个Queue,
顺序性消费的类中有一个方法会周期性地请求broker端锁定Queue
public class ConsumeMessageOrderlyService implements ConsumeMessageService {
public void start() {
if (MessageModel.CLUSTERING.equals(ConsumeMessageOrderlyService.this.defaultMQPushConsumerImpl.messageModel())) {
this.scheduledExecutorService.scheduleAtFixedRate(new Runnable() {
@Override
public void run() {
ConsumeMessageOrderlyService.this.lockMQPeriodically();
}
}, 1000 * 1, ProcessQueue.REBALANCE_LOCK_INTERVAL, TimeUnit.MILLISECONDS);
}
}
}
既然有加锁,那么肯定有解锁,重平衡时如果判断到这个Queue不再属于这个消费者,便会请求broker对这个Queue进行解锁,以便其他消费者能够消费这个Queue
// 这段重平衡的代码不知道大家有没有印象
if (mq.getTopic().equals(topic)) {
if (!mqSet.contains(mq)) {
pq.setDropped(true);
// 解锁
if (this.removeUnnecessaryMessageQueue(mq, pq)) {
it.remove();
changed = true;
log.info("doRebalance, {}, remove unnecessary mq, {}", consumerGroup, mq);
}
}
@Override
public void submitConsumeRequest(
final List<MessageExt> msgs,
final ProcessQueue processQueue,
final MessageQueue messageQueue,
final boolean dispathToConsume) {
if (dispathToConsume) {
ConsumeRequest consumeRequest = new ConsumeRequest(processQueue, messageQueue);
// 提交消息处理任务
this.consumeExecutor.submit(consumeRequest);
}
}
具体的消费代码比较,我们只看比较核心的几行
class ConsumeRequest implements Runnable {
private final ProcessQueue processQueue;
private final MessageQueue messageQueue;
public ConsumeRequest(ProcessQueue processQueue, MessageQueue messageQueue) {
this.processQueue = processQueue;
this.messageQueue = messageQueue;
}
public ProcessQueue getProcessQueue() {
return processQueue;
}
public MessageQueue getMessageQueue() {
return messageQueue;
}
@Override
public void run() {
if (this.processQueue.isDropped()) {
log.warn("run, the message queue not be able to consume, because it's dropped. {}", this.messageQueue);
return;
}
// 获取到锁
final Object objLock = messageQueueLock.fetchLockObject(this.messageQueue);
synchronized (objLock) {
if (MessageModel.BROADCASTING.equals(ConsumeMessageOrderlyService.this.defaultMQPushConsumerImpl.messageModel())
|| (this.processQueue.isLocked() && !this.processQueue.isLockExpired())) {
final long beginTime = System.currentTimeMillis();
for (boolean continueConsume = true; continueConsume; ) {
......
// 每次默认消费一个
final int consumeBatchSize =
ConsumeMessageOrderlyService.this.defaultMQPushConsumer.getConsumeMessageBatchMaxSize();
......
// 进行消费
status = messageListener.consumeMessage(Collections.unmodifiableList(msgs), context);
// 处理消费结果
continueConsume = ConsumeMessageOrderlyService.this.processConsumeResult(msgs, status, context, this);
}
}
}
}