Rocketmq--Broker写入消息源码分析

Broker通启动一个netty服务器进行生产者消息处理。

Broker通过org.apache.rocketmq.broker.BrokerStartup#main方法启动：

public static void main(String[] args) {
    start(createBrokerController(args));
}

public static BrokerController createBrokerController(String[] args) {
    try {
        BrokerController controller = buildBrokerController(args);
        boolean initResult = controller.initialize();
        if (!initResult) {
            controller.shutdown();
            System.exit(-3);
        }
        Runtime.getRuntime().addShutdownHook(new Thread(buildShutdownHook(controller)));
        return controller;
    } catch (Throwable e) {
        e.printStackTrace();
        System.exit(-1);
    }
    return null;
}

创建BrokerController时，核心就做了两件事情：

1. 解析各种配置，创建BrokerController需要的各种配置对象：BrokerConfig、NettyServerConfig、NettyClientConfig、MessageStoreConfig；
2. 调用BrokerController对象的initialize方法，进行初始化。

public boolean initialize() throws CloneNotSupportedException {

        boolean result = this.initializeMetadata();
        if (!result) {
            return false;
        }

        result = this.initializeMessageStore();
        if (!result) {
            return false;
        }

        return this.recoverAndInitService();
    }



public boolean recoverAndInitService() throws CloneNotSupportedException {


    this.brokerMetricsManager = new BrokerMetricsManager(this);

    if (result) {

        //初始化服务端
        initializeRemotingServer();

        initializeResources();
        
       //给服务端注册处理器
        registerProcessor();

        initializeScheduledTasks();

        initialTransaction();

        initialAcl();

    }

    return result;
}


protected void initializeRemotingServer() throws CloneNotSupportedException {
  // 新建remotingServer
    this.remotingServer = new NettyRemotingServer(this.nettyServerConfig, this.clientHousekeepingService);
    NettyServerConfig fastConfig = (NettyServerConfig) this.nettyServerConfig.clone();

    int listeningPort = nettyServerConfig.getListenPort() - 2;
    if (listeningPort < 0) {
        listeningPort = 0;
    }
    fastConfig.setListenPort(listeningPort);

    this.fastRemotingServer = new NettyRemotingServer(fastConfig, this.clientHousekeepingService);
}


public void registerProcessor() {
    /*
     * SendMessageProcessor
     */
    sendMessageProcessor.registerSendMessageHook(sendMessageHookList);
    sendMessageProcessor.registerConsumeMessageHook(consumeMessageHookList);
   
  
     // 注册生产者消息处理
```
this.remotingServer.registerProcessor(RequestCode.SEND_MESSAGE, sendMessageProcessor, this.sendMessageExecutor);
```
   
}
```
@Override
public void registerProcessor(int requestCode, NettyRequestProcessor processor, ExecutorService executor) {
    ExecutorService executorThis = executor;
    if (null == executor) {
        executorThis = this.publicExecutor;
    }

    Pair<NettyRequestProcessor, ExecutorService> pair = new Pair<>(processor, executorThis);
    // 方法本地map
    this.processorTable.put(requestCode, pair);
}
```

init方法中会新建remotingServer，remotingServer是一个netty服务，注册sendMessageProcessor，接收到生产者消息后，会使用它进行处理。 Controller初始化完成后，进入start：

public static BrokerController start(BrokerController controller) {
    try {
        controller.start();

        String tip = String.format("The broker[%s, %s] boot success. serializeType=%s",
            controller.getBrokerConfig().getBrokerName(), controller.getBrokerAddr(),
            RemotingCommand.getSerializeTypeConfigInThisServer());

        if (null != controller.getBrokerConfig().getNamesrvAddr()) {
            tip += " and name server is " + controller.getBrokerConfig().getNamesrvAddr();
        }

        log.info(tip);
        System.out.printf("%s%n", tip);
        return controller;
    } catch (Throwable e) {
        e.printStackTrace();
        System.exit(-1);
    }

    return null;
}


```
public void start() throws Exception {

    this.shouldStartTime = System.currentTimeMillis() + messageStoreConfig.getDisappearTimeAfterStart();

    if (messageStoreConfig.getTotalReplicas() > 1 && this.brokerConfig.isEnableSlaveActingMaster()) {
        isIsolated = true;
    }

    if (this.brokerOuterAPI != null) {
        this.brokerOuterAPI.start();
    }

  // 这里进行生产者服务端的启动
    startBasicService();

```
}

```
protected void startBasicService() throws Exception {

    if (this.messageStore != null) {
        this.messageStore.start();
    }

    if (this.timerMessageStore != null) {
        this.timerMessageStore.start();
    }

    if (this.replicasManager != null) {
        this.replicasManager.start();
    }

    if (remotingServerStartLatch != null) {
        remotingServerStartLatch.await();
    }

//启动生产者消息处理服务
    if (this.remotingServer != null) {
        this.remotingServer.start();

        // In test scenarios where it is up to OS to pick up an available port, set the listening port back to config
        if (null != nettyServerConfig && 0 == nettyServerConfig.getListenPort()) {
            nettyServerConfig.setListenPort(remotingServer.localListenPort());
        }
    }

 ```
}

controller的start方法主要就是将上面初始化的一些服务进行启动。 进入org.apache.rocketmq.remoting.netty.NettyRemotingServer#start方法：

public void start() {
    this.defaultEventExecutorGroup = new DefaultEventExecutorGroup(nettyServerConfig.getServerWorkerThreads(),
        new ThreadFactoryImpl("NettyServerCodecThread_"));

// 新建处理器
    prepareSharableHandlers();

//初始化ServerBootstrap
    serverBootstrap.group(this.eventLoopGroupBoss, this.eventLoopGroupSelector)
        .channel(useEpoll() ? EpollServerSocketChannel.class : NioServerSocketChannel.class)
        .option(ChannelOption.SO_BACKLOG, 1024)
        .option(ChannelOption.SO_REUSEADDR, true)
        .childOption(ChannelOption.SO_KEEPALIVE, false)
        .childOption(ChannelOption.TCP_NODELAY, true)
        .localAddress(new InetSocketAddress(this.nettyServerConfig.getBindAddress(),
            this.nettyServerConfig.getListenPort()))
        .childHandler(new ChannelInitializer<SocketChannel>() {
            @Override
            public void initChannel(SocketChannel ch) {
                configChannel(ch);
            }
        });
```
addCustomConfig(serverBootstrap);
```

try {
    ChannelFuture sync = serverBootstrap.bind().sync();
    InetSocketAddress addr = (InetSocketAddress) sync.channel().localAddress();
    if (0 == nettyServerConfig.getListenPort()) {
        this.nettyServerConfig.setListenPort(addr.getPort());
    }
    log.info("RemotingServer started, listening {}:{}", this.nettyServerConfig.getBindAddress(),
        this.nettyServerConfig.getListenPort());
     // 将server放到本地map
    this.remotingServerTable.put(this.nettyServerConfig.getListenPort(), this);
} catch (Exception e) {
    throw new IllegalStateException(String.format("Failed to bind to %s:%d", nettyServerConfig.getBindAddress(),
        nettyServerConfig.getListenPort()), e);
}
```
```
private void prepareSharableHandlers() {
    tlsModeHandler = new TlsModeHandler(TlsSystemConfig.tlsMode);
    encoder = new NettyEncoder();
    connectionManageHandler = new NettyConnectManageHandler();
    // 请求处理器
    serverHandler = new NettyServerHandler();
    distributionHandler = new RemotingCodeDistributionHandler();
}
```
```
public class NettyServerHandler extends SimpleChannelInboundHandler<RemotingCommand> {

    @Override
    protected void channelRead0(ChannelHandlerContext ctx, RemotingCommand msg) {
        int localPort = RemotingHelper.parseSocketAddressPort(ctx.channel().localAddress());
        NettyRemotingAbstract remotingAbstract = 
        //从上面的map取出server
        NettyRemotingServer.this.remotingServerTable.get(localPort);
        if (localPort != -1 && remotingAbstract != null) {
        // 处理逻辑
            remotingAbstract.processMessageReceived(ctx, msg);
            return;
        }
        // The related remoting server has been shutdown, so close the connected channel
        RemotingHelper.closeChannel(ctx.channel());
    }


}
```
public void processMessageReceived(ChannelHandlerContext ctx, RemotingCommand msg) {
    if (msg != null) {
        switch (msg.getType()) {
        //生产者消息处理
            case REQUEST_COMMAND:
                processRequestCommand(ctx, msg);
                break;
            case RESPONSE_COMMAND:
                processResponseCommand(ctx, msg);
                break;
            default:
                break;
        }
    }
}
```
```
public void processRequestCommand(final ChannelHandlerContext ctx, final RemotingCommand cmd) {
   // 从上面的本地map中获取处理器
   final Pair<NettyRequestProcessor, ExecutorService> matched = 
    this.processorTable.get(cmd.getCode());
    final Pair<NettyRequestProcessor, ExecutorService> pair = null == matched ? this.defaultRequestProcessorPair : matched;
   ```
   // 处理逻辑
Runnable run = buildProcessRequestHandler(ctx, cmd, pair, opaque);
```

    try {
        final RequestTask requestTask = new RequestTask(run, ctx.channel(), cmd);
        //async execute task, current thread return directly
        // 线程池异步处理请求
        pair.getObject2().submit(requestTask);
    } catch (RejectedExecutionException e) {
        if ((System.currentTimeMillis() % 10000) == 0) {
            log.warn(RemotingHelper.parseChannelRemoteAddr(ctx.channel())
                + ", too many requests and system thread pool busy, RejectedExecutionException "
                + pair.getObject2().toString()
                + " request code: " + cmd.getCode());
        }


    }
}


```
private Runnable buildProcessRequestHandler(ChannelHandlerContext ctx, RemotingCommand cmd,
    Pair<NettyRequestProcessor, ExecutorService> pair, int opaque) {
    return () -> {
        Exception exception = null;
        RemotingCommand response;

        try {
          
            if (exception == null) {
            // 处理逻辑
                response = pair.getObject1().processRequest(ctx, cmd);
            } else {
                response = RemotingCommand.createResponseCommand(RemotingSysResponseCode.SYSTEM_ERROR, null);
            }

         
}

可以看到start方法启动了一个netty服务器，并且将NettyServerHandler作为处理器，处理生产者的消息，而NettyServerHandler调用了前面的remotingServer的processMessageReceived方法，processMessageReceived方法则是将请求封装成Runnable对象，放到线程池异步执行，我们看看Runnable对象的逻辑， 进入org.apache.rocketmq.broker.processor.SendMessageProcessor#sendMessage：

```省略其他逻辑

if (brokerController.getBrokerConfig().isAsyncSendEnable()) {
    CompletableFuture<PutMessageResult> asyncPutMessageFuture;
        if (sendTransactionPrepareMessage) {
        //事务消息
            asyncPutMessageFuture = this.brokerController.getTransactionalMessageService().asyncPrepareMessage(msgInner);
        } else {
        //非事务消息
            asyncPutMessageFuture = this.brokerController.getMessageStore().asyncPutMessage(msgInner);
    }

    final int finalQueueIdInt = queueIdInt;
    final MessageExtBrokerInner finalMsgInner = msgInner;
    asyncPutMessageFuture.thenAcceptAsync(putMessageResult -> {
        RemotingCommand responseFuture =
            handlePutMessageResult(putMessageResult, response, request, finalMsgInner, responseHeader, sendMessageContext,
                ctx, finalQueueIdInt, beginTimeMillis, mappingContext, BrokerMetricsManager.getMessageType(requestHeader));
        if (responseFuture != null) {
            doResponse(ctx, request, responseFuture);
        }
```


```
public CompletableFuture<PutMessageResult> asyncPutMessage(MessageExtBrokerInner msg) {

    for (PutMessageHook putMessageHook : putMessageHookList) {
        PutMessageResult handleResult = putMessageHook.executeBeforePutMessage(msg);
        if (handleResult != null) {
            return CompletableFuture.completedFuture(handleResult);
        }
    }

    if (msg.getProperties().containsKey(MessageConst.PROPERTY_INNER_NUM)
        && !MessageSysFlag.check(msg.getSysFlag(), MessageSysFlag.INNER_BATCH_FLAG)) {
        LOGGER.warn("[BUG]The message had property {} but is not an inner batch", MessageConst.PROPERTY_INNER_NUM);
        return CompletableFuture.completedFuture(new PutMessageResult(PutMessageStatus.MESSAGE_ILLEGAL, null));
    }

    if (MessageSysFlag.check(msg.getSysFlag(), MessageSysFlag.INNER_BATCH_FLAG)) {
        Optional<TopicConfig> topicConfig = this.getTopicConfig(msg.getTopic());
        if (!QueueTypeUtils.isBatchCq(topicConfig)) {
            LOGGER.error("[BUG]The message is an inner batch but cq type is not batch cq");
            return CompletableFuture.completedFuture(new PutMessageResult(PutMessageStatus.MESSAGE_ILLEGAL, null));
        }
    }

    long beginTime = this.getSystemClock().now();
    // 调用commitLog.asyncPutMessage(msg)
    CompletableFuture<PutMessageResult> putResultFuture = this.commitLog.asyncPutMessage(msg);

    putResultFuture.thenAccept(result -> {
        long elapsedTime = this.getSystemClock().now() - beginTime;
        if (elapsedTime > 500) {
            LOGGER.warn("DefaultMessageStore#putMessage: CommitLog#putMessage cost {}ms, topic={}, bodyLength={}",
                elapsedTime, msg.getTopic(), msg.getBody().length);
        }
        this.storeStatsService.setPutMessageEntireTimeMax(elapsedTime);

        if (null == result || !result.isOk()) {
            this.storeStatsService.getPutMessageFailedTimes().add(1);
        }
    });

    return putResultFuture;
}
```

消息处理有事务消息和非事务消息，这次先看非事务消息， 进入org.apache.rocketmq.store.CommitLog#asyncPutMessage：

public CompletableFuture<PutMessageResult> asyncPutMessage(final MessageExtBrokerInner msg) {
   
  ```省略一些set逻辑
   
    MappedFile unlockMappedFile = null;
    MappedFile mappedFile = this.mappedFileQueue.getLastMappedFile();

    long currOffset;
    // 第一次写入
    if (mappedFile == null) {
        currOffset = 0;
    } else {
    // 否则从上次的位置写入
        currOffset = mappedFile.getFileFromOffset() + mappedFile.getWrotePosition();
    }

    int needAckNums = this.defaultMessageStore.getMessageStoreConfig().getInSyncReplicas();
    boolean needHandleHA = needHandleHA(msg);
  // 主从复制的逻辑，先不看
    if (needHandleHA && this.defaultMessageStore.getBrokerConfig().isEnableControllerMode()) {
        if (this.defaultMessageStore.getHaService().inSyncReplicasNums(currOffset) < this.defaultMessageStore.getMessageStoreConfig().getMinInSyncReplicas()) {
            return CompletableFuture.completedFuture(new PutMessageResult(PutMessageStatus.IN_SYNC_REPLICAS_NOT_ENOUGH, null));
        }
        if (this.defaultMessageStore.getMessageStoreConfig().isAllAckInSyncStateSet()) {
            // -1 means all ack in SyncStateSet
            needAckNums = MixAll.ALL_ACK_IN_SYNC_STATE_SET;
        }
    } else if (needHandleHA && this.defaultMessageStore.getBrokerConfig().isEnableSlaveActingMaster()) {
        int inSyncReplicas = Math.min(this.defaultMessageStore.getAliveReplicaNumInGroup(),
            this.defaultMessageStore.getHaService().inSyncReplicasNums(currOffset));
        needAckNums = calcNeedAckNums(inSyncReplicas);
        if (needAckNums > inSyncReplicas) {
            // Tell the producer, don't have enough slaves to handle the send request
            return CompletableFuture.completedFuture(new PutMessageResult(PutMessageStatus.IN_SYNC_REPLICAS_NOT_ENOUGH, null));
        }
    }
 // 加锁
    topicQueueLock.lock(topicQueueKey);
    

try {

    boolean needAssignOffset = true;
    if (defaultMessageStore.getMessageStoreConfig().isDuplicationEnable()
        && defaultMessageStore.getMessageStoreConfig().getBrokerRole() != BrokerRole.SLAVE) {
        needAssignOffset = false;
    }
    if (needAssignOffset) {
        defaultMessageStore.assignOffset(msg);
    }

    PutMessageResult encodeResult = putMessageThreadLocal.getEncoder().encode(msg);
    if (encodeResult != null) {
        return CompletableFuture.completedFuture(encodeResult);
    }
    msg.setEncodedBuff(putMessageThreadLocal.getEncoder().getEncoderBuffer());
    PutMessageContext putMessageContext = new PutMessageContext(topicQueueKey);

    putMessageLock.lock(); //spin or ReentrantLock ,depending on store config
    try {
        long beginLockTimestamp = this.defaultMessageStore.getSystemClock().now();
        this.beginTimeInLock = beginLockTimestamp;

        // Here settings are stored timestamp, in order to ensure an orderly
        // global
        if (!defaultMessageStore.getMessageStoreConfig().isDuplicationEnable()) {
            msg.setStoreTimestamp(beginLockTimestamp);
        }

        if (null == mappedFile || mappedFile.isFull()) {
            mappedFile = this.mappedFileQueue.getLastMappedFile(0); // Mark: NewFile may be cause noise
            if (isCloseReadAhead()) {
                setFileReadMode(mappedFile, LibC.MADV_RANDOM);
            }
        }
        if (null == mappedFile) {
            log.error("create mapped file1 error, topic: " + msg.getTopic() + " clientAddr: " + msg.getBornHostString());
            beginTimeInLock = 0;
            return CompletableFuture.completedFuture(new PutMessageResult(PutMessageStatus.CREATE_MAPPED_FILE_FAILED, null));
        }

  // 将消息写入内存中
            result = mappedFile.appendMessage(msg, this.appendMessageCallback, putMessageContext);
            switch (result.getStatus()) {
            // 写入成功
                case PUT_OK:
                // 空逻辑
                    onCommitLogAppend(msg, result, mappedFile);
                    break;
                    // 文件满了，从上个文件结束位置新建一个再写入
                case END_OF_FILE:
                    onCommitLogAppend(msg, result, mappedFile);
                    unlockMappedFile = mappedFile;
                    // Create a new file, re-write the message
                    mappedFile = this.mappedFileQueue.getLastMappedFile(0);
                    if (null == mappedFile) {
                        // XXX: warn and notify me
                        log.error("create mapped file2 error, topic: " + msg.getTopic() + " clientAddr: " + msg.getBornHostString());
                        beginTimeInLock = 0;
                        return CompletableFuture.completedFuture(new PutMessageResult(PutMessageStatus.CREATE_MAPPED_FILE_FAILED, result));
                    }
                    if (isCloseReadAhead()) {
                        setFileReadMode(mappedFile, LibC.MADV_RANDOM);
                    }
                    result = mappedFile.appendMessage(msg, this.appendMessageCallback, putMessageContext);
                    if (AppendMessageStatus.PUT_OK.equals(result.getStatus())) {
                        onCommitLogAppend(msg, result, mappedFile);
                    }
                    break;
                case MESSAGE_SIZE_EXCEEDED:
                case PROPERTIES_SIZE_EXCEEDED:
                    beginTimeInLock = 0;
                    return CompletableFuture.completedFuture(new PutMessageResult(PutMessageStatus.MESSAGE_ILLEGAL, result));
                case UNKNOWN_ERROR:
                default:
                    beginTimeInLock = 0;
                    return CompletableFuture.completedFuture(new PutMessageResult(PutMessageStatus.UNKNOWN_ERROR, result));
            }


```

```
// Statistics
storeStatsService.getSinglePutMessageTopicTimesTotal(msg.getTopic()).add(result.getMsgNum());
storeStatsService.getSinglePutMessageTopicSizeTotal(topic).add(result.getWroteBytes());
// 同步或异步将消息刷盘
return handleDiskFlushAndHA(putMessageResult, msg, needAckNums, needHandleHA);
```
```

消息的存储是用commitlog来表示的，所有的消息都放到一个commitlog目录中，每个commitlog大小为1GB，满了则新建，这里有两个锁，topicQueueLock是topic+queue粒度的， defaultMessageStore.assignOffset(msg)会分配offset，同一个topic的同一个queue要用同一把锁，防止乱序， putMessageLock是用来锁文件的，因为commitlog是多线程写入的，不同的topic或者queue可能同时写，所以这里也需要一个锁（个人感觉是不是只要 putMessageLock这一个锁就够了，放在topicQueueLock的位置）， asyncPutMessage方法就是先将消息写到内存映射文件中，然后同步或异步进行消息刷盘。