RocketMQ部署及使用-Go （二）

黑大天之骄子
110 阅读25分钟

篇章（二）以rocketmq 4.x版本为基础，对4.x不感兴趣的可以看篇章（三）哈。

生产者

Tag

不同的 Topic 之间的消息没有必然的联系，而 Tag 则用来区分同一个 Topic 下相互关联的消息，例如全集和子集的关系

Keys

每个消息可以在业务层面的设置唯一标识码 keys 字段，方便将来定位消息丢失问题。

队列

为了支持高并发和水平扩展，需要对 Topic 进行分区，在 RocketMQ 中这被称为队列。

消息在队列中按照先进先出的原则存储，每条消息会有自己的位点，每个队列会统计当前消息的总条数，称为最大位点 MaxOffset；队列的起始位置对应的位置叫做起始位点 MinOffset。队列可以提升消息发送和消费的并发度。 image.png

消息类型

  • 普通消息
  • 顺序消息
  • 延迟消息
  • 批量消息
  • 事务消息

普通消息发送

发送模式

  • 同步发送

    同步发送等待结果最后返回SendResul

    package main

import (
    "context"
    "fmt"
    "os"
    "strconv"

    "github.com/apache/rocketmq-client-go/v2"
    "github.com/apache/rocketmq-client-go/v2/primitive"
    "github.com/apache/rocketmq-client-go/v2/producer"
)

// Package main implements a simple producer to send message.
func main() {
    topic := "test-simple-topic"
    p, _ := rocketmq.NewProducer(
        producer.WithNameServer([]string{"127.0.0.1:9876"}),
        producer.WithRetry(2),
    )
    err := p.Start()
    if err != nil {
        fmt.Printf("start producer error: %s", err.Error())
        os.Exit(1)
    }

    for i := 0; i < 10; i++ {
        msg := &primitive.Message{
            Topic: topic,
            Body:  []byte("Hello RocketMQ Go Client! " + strconv.Itoa(i)),
        }
        // sync send
        res, err := p.SendSync(context.Background(), msg)
        if err != nil {
            fmt.Printf("send message error: %s\n", err)
        } else {
            fmt.Printf("send message success: result=%s\n", res.String())
        }
    }
    err = p.Shutdown()
    if err != nil {
        fmt.Printf("shutdown producer error: %s", err.Error())
    }
}

  • 异步发送

  • 异步发送不会等待发送返回，异步发送需要实现异步发送回调接口（SendCallback）。

    package main

import (
    "context"
    "fmt"
    "os"
    "sync"

    "github.com/apache/rocketmq-client-go/v2"
    "github.com/apache/rocketmq-client-go/v2/primitive"
    "github.com/apache/rocketmq-client-go/v2/producer"
)

// Package main implements a async producer to send message.
func main() {
    topic := "test-async-topic"
    p, _ := rocketmq.NewProducer(
        producer.WithNameServer([]string{"127.0.0.1:9876"}),
        producer.WithRetry(2))

    err := p.Start()
    if err != nil {
        fmt.Printf("start producer error: %s", err.Error())
        os.Exit(1)
    }
    var wg sync.WaitGroup
    for i := 0; i < 10; i++ {
        wg.Add(1)
        // async send
        err := p.SendAsync(context.Background(),
            func(ctx context.Context, result *primitive.SendResult, e error) {
                if e != nil {
                    fmt.Printf("receive message error: %s\n", err)
                } else {
                    fmt.Printf("send message success: result=%s\n", result.String())
                }
                wg.Done()
            },
            primitive.NewMessage(topic, []byte(fmt.Sprintf("Hello RocketMQ Go Client:%d", i))),
        )
        if err != nil {
            fmt.Printf("send message error: %s\n", err)
        }

        // send one way
        // p.SendOneWay(context.Background(), primitive.NewMessage(topic, []byte("Hello RocketMQ Go Client!")))
    }
    wg.Wait()
    err = p.Shutdown()
    if err != nil {
        fmt.Printf("shutdown producer error: %s", err.Error())
    }

    select {}
}

  • 单向发送

    单向模式调用sendOneway，不会对返回结果有任何等待和处理。

    package main

import (
    "context"
    "fmt"
    "os"
    "sync"

    "github.com/apache/rocketmq-client-go/v2"
    "github.com/apache/rocketmq-client-go/v2/primitive"
    "github.com/apache/rocketmq-client-go/v2/producer"
)

// Package main implements a async producer to send message.
func main() {
    topic := "test-async-topic"
    p, _ := rocketmq.NewProducer(
        producer.WithNameServer([]string{"127.0.0.1:9876"}),
        producer.WithRetry(2))

    err := p.Start()
    if err != nil {
        fmt.Printf("start producer error: %s", err.Error())
        os.Exit(1)
    }
    for i := 0; i < 10; i++ {
        // send one way
        // p.SendOneWay(context.Background(), primitive.NewMessage(topic, []byte("Hello RocketMQ Go Client!")))
    }
    err = p.Shutdown()
    if err != nil {
        fmt.Printf("shutdown producer error: %s", err.Error())
    }

    select {}
}

顺序消息

介绍

顺序消息是一种对消息发送和消费顺序有严格要求的消息，生产顺序性和消费顺序性。只有同时满足了生产顺序性和消费顺序性才能达到FIFO效果。

对于一个指定的Topic，消息严格按照先进先出（FIFO）的原则进行消息发布和消费，即先发布的消息先消费，后发布的消息后消费。

RocketMQ 中支持分区顺序消息，如下图所示。我们可以按照某一个标准对消息进行分区（比如图中的ShardingKey），同一个ShardingKey的消息会被分配到同一个队列中，并按照顺序被消费。

  • 生产顺序性

    • 单一生产者： 消息生产的顺序性仅支持单一生产者，不同生产者分布在不同的系统，即使设置相同的分区键，不同生产者之间产生的消息也无法判定其先后顺序。
    • 串行发送：生产者客户端支持多线程安全访问，但如果生产者使用多线程并行发送，则不同线程间产生的消息将无法判定其先后顺序。 image.png

  • 消费顺序性

    • 在消费者篇章讲到。
顺序消息的一致性保障

如果一个Broker掉线，那么此时队列总数可能发生变化，同一个 ShardingKey 的消息就会发送到不同的队列上，造成乱序。RocketMQ 提供了两种模式：

  • 保证严格顺序，创建 Topic 是要指定 -o 参数（--order）为true，表示顺序消息: 
    sh bin/mqadmin updateTopic -c DefaultCluster -t TopicTest -o true -n 127.0.0.1:9876

保证NameServer中的配置 orderMessageEnable 和 returnOrderTopicConfigToBroker 必须是 true
  • 任意一个条件不满足，则是保证可用性而不是严格顺序。
package main

import (
    "context"
    "fmt"
    "github.com/apache/rocketmq-client-go/v2"
    "github.com/apache/rocketmq-client-go/v2/primitive"
    "github.com/apache/rocketmq-client-go/v2/producer"
    "strconv"
)
func main() {
    topic := "test-order-topic"
    p, err := rocketmq.NewProducer(
        producer.WithNameServer([]string{"127.0.0.1:9876"}),
        producer.WithRetry(2),
        producer.WithQueueSelector(producer.NewHashQueueSelector()), 
    )
    if err != nil {
        panic(err)
    }
    err = p.Start()
    if err != nil {
        panic(err)
    }
    for i := 0; i < 3; i++ {
        orderId := strconv.Itoa(i)
        for j := 0; j < 5; j++ {
            msg := &primitive.Message{
                Topic: topic,
                Body:  []byte("Ordered Message Step -> " + strconv.Itoa(j)),
            }
            msg.WithShardingKey(orderId)
            res, err := p.SendSync(context.Background(), msg)
            if err != nil {
                fmt.Errorf("send message success: result=%s\n", res.String())
                continue
            }

            fmt.Printf("send message success: result=%s\n", res.String())
        }
    }
    // close producer
    err = p.Shutdown()
    if err != nil {
        fmt.Printf("shutdown producer error: %s\n", err.Error())
    }
}


package main

import (
    "context"
    "fmt"
    "github.com/apache/rocketmq-client-go/v2"
    "github.com/apache/rocketmq-client-go/v2/consumer"
    "github.com/apache/rocketmq-client-go/v2/primitive"
    "github.com/sirupsen/logrus"
    "os"
    "os/signal"
)

func main() {
    topic := "test-order-topic"
    c, err := rocketmq.NewPushConsumer(
        consumer.WithGroupName("testGroup"),
        consumer.WithNameServer([]string{"127.0.0.1:9876"}),
        consumer.WithConsumerOrder(true),
        consumer.WithConsumerModel(consumer.Clustering),
    )
    if err != nil {
        panic(err)
    }
    err = c.Subscribe(topic, consumer.MessageSelector{}, func(ctx context.Context,
        msgs ...*primitive.MessageExt) (consumer.ConsumeResult, error) {
        for i := range msgs {
            logrus.Infof("Receive message: %s, orderId: %s", msgs[i].Body, msgs[i].GetShardingKey())
        }
        return consumer.ConsumeSuccess, nil
    })
    if err != nil {
        panic(err)
    }
    // Note: start after subscribe
    err = c.Start()
    if err != nil {
        panic(err)
    }
    logrus.Info("Consumer Started.")
    stop := make(chan os.Signal)
    signal.Notify(stop, os.Interrupt, os.Kill)
    select {
    case <-stop:
        err = c.Shutdown()
        if err != nil {
            fmt.Printf("shutdown Consumer error: %s", err.Error())
        }
    }
}

延迟消息发送

延迟消息发送是指消息发送到Apache RocketMQ后，并不期望立马投递这条消息，而是延迟一定时间后才投递到Consumer进行消费。 image.png

延时消息约束

image.png

package main

import (
    "context"
    "fmt"
    "github.com/apache/rocketmq-client-go/v2"
    "github.com/apache/rocketmq-client-go/v2/primitive"
    "github.com/apache/rocketmq-client-go/v2/producer"
    "os"
)

func main() {
    topic := "test-delay-topic"
    p, err := rocketmq.NewProducer(
        producer.WithNameServer([]string{"127.0.0.1:9876"}),
        producer.WithRetry(2),
    )
    err = p.Start()
    if err != nil {
        fmt.Printf("start producer error: %s", err.Error())
        os.Exit(1)
    }
    for i := 0; i < 10; i++ {
        msg := primitive.NewMessage(topic, []byte(fmt.Sprintf("Hello RocketMQ Go Client:%v", i)))
        msg.WithDelayTimeLevel(3)
        res, err := p.SendSync(context.Background(), msg)
        if err != nil {
            fmt.Printf("send message error: %s\n", err)
        } else {
            fmt.Printf("send message success: result=%s\n", res.String())
        }
    }
    err = p.Shutdown()
    if err != nil {
        fmt.Printf("shutdown producer error: %s", err.Error())
    }
}


package main

import (
    "context"
    "fmt"
    "github.com/apache/rocketmq-client-go/v2"
    "github.com/apache/rocketmq-client-go/v2/consumer"
    "github.com/apache/rocketmq-client-go/v2/primitive"
    "os"
    "time"
)

func main() {
    topic := "test-delay-topic"
    c, err := rocketmq.NewPushConsumer(
        consumer.WithGroupName("testGroup"),
        consumer.WithNameServer([]string{"127.0.0.1:9876"}),
    )
    err = c.Subscribe(topic, consumer.MessageSelector{},
        func(ctx context.Context, msgs ...*primitive.MessageExt) (consumer.ConsumeResult, error) {
            for _, msg := range msgs {
                t := time.Now().UnixNano()/int64(time.Millisecond) - msg.BornTimestamp
                fmt.Printf("Receive message[msgId=%s] %d ms later\n", msg.MsgId, t)
            }
            return consumer.ConsumeSuccess, nil
        })
    if err != nil {
        fmt.Println(err.Error())
    }
    // Note: start after subscribe
    err = c.Start()
    if err != nil {
        fmt.Println(err.Error())
        os.Exit(-1)
    }
    time.Sleep(time.Hour)
    err = c.Shutdown()
    if err != nil {
        fmt.Printf("Shutdown Consumer error: %s", err.Error())
    }
}

批量消息

在对吞吐率有一定要求的情况下，Apache RocketMQ可以将一些消息聚成一批以后进行发送，可以增加吞吐率，并减少API和网络调用次数。

需要注意的是批量消息的大小不能超过 1MiB（否则需要自行分割），其次同一批 batch 中 topic 必须相同。 image.png

  • 批量消息模式下，同一批次消息会发送至同一queue
send message success: result=SendResult
[sendStatus=0, msgIds=, 
offsetMsgId=C0A801AA00002A9F0000000000015B42,
C0A801AA00002A9F0000000000015BE3,
C0A801AA00002A9F0000000000015C84,
C0A801AA00002A9F0000000000015D25,
C0A801AA00002A9F0000000000015DC6,
C0A801AA00002A9F0000000000015E67,
C0A801AA00002A9F0000000000015F08,
C0A801AA00002A9F0000000000015FA9,
C0A801AA00002A9F000000000001604A,
C0A801AA00002A9F00000000000160EB, 
queueOffset=290, 
messageQueue=MessageQueue [topic=test-batch-topic, brokerName=ccc6d8a404d4, queueId=1]]

  • 消费者设置的集群模式没有效果，原因在于消息是批量发送到一个queue，每个topic下默认4个queue，而基于消费者的负载均衡，每个消费者会绑定一个queue。

  • 查看topic的消费者数量：

sh mqadmin topicRoute -n rmqnamesrv:9876 -t test-batch-topic

{
    "brokerDatas":[
       {
          "brokerAddrs":{0:"192.168.1.170:10911"
          },
          "brokerName":"ccc6d8a404d4",
          "cluster":"DefaultCluster"
       }
    ],
    "filterServerTable":{},
    "queueDatas":[
       {
          "brokerName":"ccc6d8a404d4",
          "perm":8,
          "readQueueNums":8,
          "topicSysFlag":0,
          "writeQueueNums":4
       }
    ]
}
  • 修改topic的queue数量:
sh mqadmin updateTopic -n 127.0.0.1:9876 -t YourTopicName -q 8 -c DefaultCluster

package main
import (
    "context"
    "fmt"
    "github.com/apache/rocketmq-client-go/v2"
    "github.com/apache/rocketmq-client-go/v2/primitive"
    "github.com/apache/rocketmq-client-go/v2/producer"
    "os"
    "strconv"
)

func main() {
    topic := "test-batch-topic"
    p, err := rocketmq.NewProducer(
       producer.WithNameServer([]string{"127.0.0.1:9876"}),
       producer.WithRetry(2),
    )
    err = p.Start()
    if err != nil {
       fmt.Printf("start producer error: %s", err.Error())
       os.Exit(1)
    }
    var msgs []*primitive.Message
    for i := 0; i < 1000; i++ {
       msgs = append(msgs, primitive.NewMessage(topic, []byte("Hello RocketMQ Go Client! num: "+strconv.Itoa(i))))
    }
    res, err := p.SendSync(context.Background(), msgs...)
    if err != nil {
       fmt.Printf("send message error: %s\n", err)
    } else {
       fmt.Printf("send message success: result=%s\n", res.String())
    }
    err = p.Shutdown()
    if err != nil {
       fmt.Printf("shutdown producer error: %s", err.Error())
    }
}



package main

import (
    "context"
    "fmt"
    "github.com/apache/rocketmq-client-go/v2"
    "github.com/apache/rocketmq-client-go/v2/consumer"
    "github.com/apache/rocketmq-client-go/v2/primitive"
    "os"
    "time"
)

func main() {
    topic := "test-batch-topic"
    c, err := rocketmq.NewPushConsumer(
        consumer.WithGroupName("testGroup"),
        consumer.WithNameServer([]string{"127.0.0.1:9876"}),
        consumer.WithConsumeMessageBatchMaxSize(5), // 设置批量消费，每次最多拉取 5 条消息
        consumer.WithConsumerModel(consumer.Clustering),
    )
    err = c.Subscribe(topic,
        consumer.MessageSelector{},
        func(ctx context.Context, msgs ...*primitive.MessageExt) (consumer.ConsumeResult, error) {
            fmt.Printf("收到批量消息: 本次共%d条\n", len(msgs))
            for _, msg := range msgs {
                fmt.Printf("Message ID: %s, Body: %s\n", msg.MsgId, string(msg.Body))
            }
            return consumer.ConsumeSuccess, nil
        },
    )
    if err != nil {
        fmt.Println(err.Error())
    }
    err = c.Start()
    if err != nil {
        fmt.Println(err.Error())
        os.Exit(-1)
    }
    time.Sleep(time.Hour)
    err = c.Shutdown()
    if err != nil {
        fmt.Printf("Shutdown Consumer error: %s", err.Error())
    }
}

事务消息发送

事务消息介绍

在一些对数据一致性有强需求的场景，可以用 Apache RocketMQ 事务消息来解决，从而保证上下游数据的一致性。 image.png

以电商交易场景为例，用户支付订单这一核心操作的同时会涉及到下游物流发货、积分变更、购物车状态清空等多个子系统的变更。当前业务的处理分支包括：

  • 主分支订单系统状态更新：由未支付变更为支付成功。
  • 物流系统状态新增：新增待发货物流记录，创建订单物流记录。
  • 积分系统状态变更：变更用户积分，更新用户积分表。
  • 购物车系统状态变更：清空购物车，更新用户购物车记录。 image.png 使用普通消息无法保证一致的原因，本质上是由于普通消息无法像单机数据库事务一样，具备提交、回滚和统一协调的能力。 而基于 RocketMQ 的分布式事务消息功能，在普通消息基础上，支持二阶段的提交能力。将二阶段提交和本地事务绑定，实现全局提交结果的一致性。

事务消息发送分为两个阶段:

  • 第一阶段会发送一个半事务消息，半事务消息是指暂不能投递的消息，生产者已经成功地将消息发送到了 Broker，但是Broker 未收到生产者对该消息的二次确认，此时该消息被标记成“暂不能投递”状态。
  • 如果发送成功则执行本地事务，并根据本地事务执行成功与否，向 Broker 半事务消息状态（commit或者rollback），半事务消息只有 commit 状态才会真正向下游投递。
  • 如果由于网络闪断、生产者应用重启等原因，导致某条事务消息的二次确认丢失，Broker 端会通过扫描发现某条消息长期处于“半事务消息”时，需要主动向消息生产者询问该消息的最终状态（Commit或是Rollback）。这样最终保证了本地事务执行成功，下游就能收到消息，本地事务执行失败，下游就收不到消息。总而保证了上下游数据的一致性。 image.png

事务消息发送步骤如下：

  • 生产者将半事务消息发送至 RocketMQ Broker。
  • RocketMQ Broker 将消息持久化成功之后，向生产者返回 Ack 确认消息已经发送成功，此时消息暂不能投递，为半事务消息。
  • 生产者开始执行本地事务逻辑。
  • 生产者根据本地事务执行结果向服务端提交二次确认结果（Commit或是Rollback），服务端收到确认结果后处理逻辑如下： 二次确认结果为Commit：服务端将半事务消息标记为可投递，并投递给消费者。 二次确认结果为Rollback：服务端将回滚事务，不会将半事务消息投递给消费者。
  • 在断网或者是生产者应用重启的特殊情况下，若服务端未收到发送者提交的二次确认结果，或服务端收到的二次确认结果为Unknown未知状态，经过固定时间后，服务端将对消息生产者即生产者集群中任一生产者实例发起消息回查。服务端仅仅会按照参数尝试指定次数，超过次数后事务会强制回滚，因此未决事务的回查时效性非常关键，需要按照业务的实际风险来设置
  • 生产者收到消息回查后，需要检查对应消息的本地事务执行的最终结果。
  • 生产者根据检查得到的本地事务的最终状态再次提交二次确认，服务端仍按照步骤4对半事务消息进行处理。

需要注意的是事务消息的生产组名称 ProducerGroupName不能随意设置。事务消息有回查机制，回查时Broker端如果发现原始生产者已经崩溃，则会联系同一生产者组的其他生产者实例回查本地事务执行情况以Commit或Rollback半事务消息。

package main

import (
    "context"
    "fmt"
    "os"
    "strconv"
    "sync"
    "sync/atomic"
    "time"

    "github.com/apache/rocketmq-client-go/v2"
    "github.com/apache/rocketmq-client-go/v2/primitive"
    "github.com/apache/rocketmq-client-go/v2/producer"
)

type DemoListener struct {
    localTrans       *sync.Map
    transactionIndex int32
}

func NewDemoListener() *DemoListener {
    return &DemoListener{
       localTrans: new(sync.Map),
    }
}

// - 制触发事务回查机制
// 虽然计算了不同的状态并存储在 localTrans 中，但返回 UnknowState 会让 RocketMQ 服务器后续通过 CheckLocalTransaction 来回查事务状态
func (dl *DemoListener) ExecuteLocalTransaction(msg *primitive.Message) primitive.LocalTransactionState {
    nextIndex := atomic.AddInt32(&dl.transactionIndex, 1)
    fmt.Printf("nextIndex: %v for transactionID: %v\n", nextIndex, msg.TransactionId)
    dl.localTrans.Store(msg.TransactionId, primitive.LocalTransactionState(primitive.UnknowState))
    return primitive.UnknowState
}

func (dl *DemoListener) CheckLocalTransaction(msg *primitive.MessageExt) primitive.LocalTransactionState {
    fmt.Printf("%v msg transactionID : %v\n", time.Now(), msg.TransactionId)
    v, existed := dl.localTrans.Load(msg.TransactionId)
    if !existed {
       fmt.Printf("unknown msg: %v, return Commit", msg)
       return primitive.RollbackMessageState
    }
    state := v.(primitive.LocalTransactionState)
    switch state {
    case 1:
       fmt.Printf("checkLocalTransaction COMMIT_MESSAGE: %v\n", msg)
       return primitive.CommitMessageState
    case 2:
       fmt.Printf("checkLocalTransaction ROLLBACK_MESSAGE: %v\n", msg)
       return primitive.RollbackMessageState
    case 3:
       fmt.Printf("checkLocalTransaction unknown: %v\n", msg)
       return primitive.UnknowState
    default:
       fmt.Printf("checkLocalTransaction default ROLLBACK_MESSAGE: %v\n", msg)
       return primitive.RollbackMessageState
    }
}

func main() {
    p, _ := rocketmq.NewTransactionProducer(
       NewDemoListener(),
       producer.WithNameServer([]string{"127.0.0.1:9876"}),
       producer.WithRetry(1),
       producer.WithGroupName("test_transaction_producer_group"),
    )
    err := p.Start()
    if err != nil {
       fmt.Printf("start producer error: %s\n", err.Error())
       os.Exit(1)
    }

    for i := 0; i < 3; i++ {
       res, err := p.SendMessageInTransaction(context.Background(),
          primitive.NewMessage("test_transaction_topic", []byte("Hello RocketMQ again "+strconv.Itoa(i))))

       if err != nil {
          fmt.Printf("send message error: %s\n", err)
       } else {
          fmt.Printf("send message success: result=%s\n", res.String())
       }
    }
    
    select {}
}


package main

import (
    "context"
    "fmt"
    "os"
    "time"

    "github.com/apache/rocketmq-client-go/v2"
    "github.com/apache/rocketmq-client-go/v2/consumer"
    "github.com/apache/rocketmq-client-go/v2/primitive"
)

func main() {
    c, _ := rocketmq.NewPushConsumer(
       consumer.WithNameServer([]string{"127.0.0.1:9876"}),
       consumer.WithGroupName("transaction_consumer_group"),
    )
    err := c.Subscribe(
       "test_transaction_topic",
       consumer.MessageSelector{},
       func(ctx context.Context, msgs ...*primitive.MessageExt) (consumer.ConsumeResult, error) {
          for i := range msgs {
             fmt.Printf("接收到消息: id=%v, topic=%v, body=%v, transactionId=%v\n",
                msgs[i].MsgId,
                msgs[i].Topic,
                string(msgs[i].Body),
                msgs[i].TransactionId)
          }
          return consumer.ConsumeSuccess, nil
       })

    if err != nil {
       fmt.Printf("订阅主题失败: %s\n", err.Error())
       os.Exit(1)
    }
    err = c.Start()
    if err != nil {
       fmt.Printf("启动消费者失败: %s\n", err.Error())
       os.Exit(1)
    }

    time.Sleep(time.Hour)
    err = c.Shutdown()
    if err != nil {
       fmt.Printf("关闭消费者失败: %s\n", err.Error())
       os.Exit(1)
    }
}


ERRO[0000] executeLocalTransaction but state unexpected  
localState=3 message="[topic=test_transaction_topic, body=Hello RocketMQ again 0, Flag=0, properties=map[PGROUP:test_transaction_producer_group TRAN_MSG:true UNIQ_KEY:C0A801C376450000000085a002000001], 
TransactionId=C0A801C376450000000085a002000001]"
send message success: result=SendResult [sendStatus=0, msgIds=C0A801C376450000000085a002000001, offsetMsgId=C0A801C300002A9F00000000000EEF51, queueOffset=122, messageQueue=MessageQueue [topic=test_transaction_topic, brokerName=ccc6d8a404d4, queueId=1]]

ERRO[0000] executeLocalTransaction but state unexpected  
localState=3 message="[topic=test_transaction_topic, body=Hello RocketMQ again 1, Flag=0, properties=map[PGROUP:test_transaction_producer_group TRAN_MSG:true UNIQ_KEY:C0A801C376450000000085a002000002], 
TransactionId=C0A801C376450000000085a002000002]"
send message success: result=SendResult [sendStatus=0, msgIds=C0A801C376450000000085a002000002, offsetMsgId=C0A801C300002A9F00000000000EF07C, queueOffset=123, messageQueue=MessageQueue [topic=test_transaction_topic, brokerName=ccc6d8a404d4, queueId=2]]

ERRO[0000] executeLocalTransaction but state unexpected  
localState=3 message="[topic=test_transaction_topic, body=Hello RocketMQ again 2, Flag=0, properties=map[PGROUP:test_transaction_producer_group TRAN_MSG:true UNIQ_KEY:C0A801C376450000000085a002000003], 
TransactionId=C0A801C376450000000085a002000003]"
send message success: result=SendResult [sendStatus=0, msgIds=C0A801C376450000000085a002000003, offsetMsgId=C0A801C300002A9F00000000000EF1A7, queueOffset=124, messageQueue=MessageQueue [topic=test_transaction_topic, brokerName=ccc6d8a404d4, queueId=3]]

2025-02-26 22:45:03.846976 +0800 CST m=+47.455887544 
msg transactionID : C0A801C376450000000085a002000003
checkLocalTransaction unknown: [Message=[topic=test_transaction_topic, body=Hello RocketMQ again 2, Flag=0, properties=map[CLUSTER:DefaultCluster PGROUP:test_transaction_producer_group REAL_QID:3 REAL_TOPIC:test_transaction_topic TRANSACTION_CHECK_TIMES:1 TRAN_MSG:true UNIQ_KEY:C0A801C376450000000085a002000003 WAIT:false], TransactionId=C0A801C376450000000085a002000003], MsgId=C0A801C376450000000085a002000003, OffsetMsgId=C0A801C300002A9F00000000000EF55C,QueueId=3, StoreSize=334, QueueOffset=127, SysFlag=0, BornTimestamp=1740581056410, BornHost=172.20.0.1:61362, StoreTimestamp=1740581056410, StoreHost=192.168.1.195:10911, CommitLogOffset=980316, BodyCRC=1908435687, ReconsumeTimes=0, PreparedTransactionOffset=0]

2025-02-26 22:45:03.847155 +0800 CST m=+47.456066182 msg transactionID : C0A801C376450000000085a002000001
checkLocalTransaction unknown: [Message=[topic=test_transaction_topic, body=Hello RocketMQ again 0, Flag=0, properties=map[CLUSTER:DefaultCluster PGROUP:test_transaction_producer_group REAL_QID:1 REAL_TOPIC:test_transaction_topic TRANSACTION_CHECK_TIMES:1 TRAN_MSG:true UNIQ_KEY:C0A801C376450000000085a002000001 WAIT:false], TransactionId=C0A801C376450000000085a002000001], MsgId=C0A801C376450000000085a002000001, OffsetMsgId=C0A801C300002A9F00000000000EF2D2,QueueId=1, StoreSize=334, QueueOffset=125, SysFlag=0, BornTimestamp=1740581056399, BornHost=172.20.0.1:61362, StoreTimestamp=1740581056401, StoreHost=192.168.1.195:10911, CommitLogOffset=979666, BodyCRC=533597131, ReconsumeTimes=0, PreparedTransactionOffset=0]

2025-02-26 22:46:03.849888 +0800 CST m=+107.457303432 msg transactionID : C0A801C376450000000085a002000002
checkLocalTransaction unknown: [Message=[topic=test_transaction_topic, body=Hello RocketMQ again 1, Flag=0, properties=map[CLUSTER:DefaultCluster PGROUP:test_transaction_producer_group REAL_QID:2 REAL_TOPIC:test_transaction_topic TRANSACTION_CHECK_TIMES:2 TRAN_MSG:true UNIQ_KEY:C0A801C376450000000085a002000002 WAIT:false], TransactionId=C0A801C376450000000085a002000002], MsgId=C0A801C376450000000085a002000002, OffsetMsgId=C0A801C300002A9F00000000000EF7E6,QueueId=2, StoreSize=334, QueueOffset=129, SysFlag=0, BornTimestamp=1740581056404, BornHost=172.20.0.1:61362, StoreTimestamp=1740581103844, StoreHost=192.168.1.195:10911, CommitLogOffset=980966, BodyCRC=1758018397, ReconsumeTimes=0, PreparedTransactionOffset=0]

消费者

消费者与消费组

消息系统的重要作用之一是削峰填谷，最简单的方式就是扩容消费者。如果多个消费者设置了相同的Consumer Group，被认为这些消费者在同一个消费组内。这与5.0版本有很大不同。

不同的消费者组可以同时订阅该topic：

topic := "test-simple-topic"
    c, _ := rocketmq.NewPushConsumer(
        consumer.WithGroupName("testGroup2"),
        consumer.WithNameServer([]string{"127.0.0.1:9876"}),
        consumer.WithConsumerModel(consumer.BroadCasting),
    )

topic := "test-simple-topic"
    c, _ := rocketmq.NewPushConsumer(
        consumer.WithGroupName("testGroup"),
        consumer.WithNameServer([]string{"127.0.0.1:9876"}),
    )

RocketMQ 有两种消费模式，分别是：

  • 集群消费模式：当使用集群消费模式时，RocketMQ 认为任意一条消息只需要被消费组内的任意一个消费者处理即可。

  • 广播消费模式：当使用广播消费模式时，RocketMQ 会将每条消息推送给消费组所有的消费者，保证消息至少被每个消费者消费一次。 image.png image.png

consumer.WithConsumerModel(consumer.BroadCasting) // 集群消费模式
consumer.WithConsumerModel(consumer.Clustering) // 广播消费模式

负载均衡

RocketMQ默认的分配策略是平均分配 image.png image.png

但也不是一味地增加消费者就能提升消费能力的，比如下图中Topic的总队列数小于消费者的数量时，消费者将分配不到队列，即使消费者再多也无法提升消费能力。 image.png

消费位点

每个队列都会记录自己的最小位点、最大位点。针对于消费组，还有消费位点的概念，

在集群模式下，消费位点是由客户端（消费者）提给交服务端保存的，

在广播模式下，消费位点是由客户端自己（消费者）保存的。一般情况下消费位点正常更新，不会出现消息重复，但如果消费者发生崩溃或有新的消费者加入群组，就会触发重平衡，重平衡完成后，每个消费者可能会分配到新的队列，而不是之前处理的队列。为了能继续之前的工作，消费者需要读取每个队列最后一次的提交的消费位点，然后从消费位点处继续拉取消息。但在实际执行过程中，由于客户端提交给服务端的消费位点并不是实时的，所以重平衡就可能会导致消息少量重复。 image.png

如何解决消息重复？

利用 RocketMQ 自身机制
  • 消息唯一标识和消费者确认机制：消费者需要显式地向 Broker 发送 ACK 来确认消息已经被成功处理。消费者在处理消息前先获取消息的唯一标识，处理完成后将该标识和处理结果记录下来，再发送 ACK。当新消息到达时，先检查 ID 是否已处理过，若已处理则跳过，从而避免重复消费。
  • 事务消息机制：对于事务消息，RocketMQ 采用半消息和反查机制来保证消息的最终一致性。如果事务消息提交失败，RocketMQ 会回滚并尝试取消这条消息，从而避免了消息重复的问题。

推、拉和长轮询

消费模式可以大致分为两种，一种是推Push，一种是拉Pull。

Push是服务端主动推送消息给客户端，优点是及时性较好，但如果客户端没有做好流控，一旦服务端推送大量消息到客户端时，就会导致客户端消息堆积甚至崩溃。

Pull是客户端需要主动到服务端取数据，优点是客户端可以依据自己的消费能力进行消费，但拉取的频率也需要用户自己控制，拉取频繁容易造成服务端和客户端的压力，拉取间隔长又容易造成消费不及时。

Push消费

顺序消费

并发消费中，可能会有多个线程同时消费一个队列的消息，因此即使发送端通过发送顺序消息保证消息在同一个队列中按照FIFO的顺序，也无法保证消息实际被顺序消费。

consumer.WithConsumerOrder(true)

并发消费

consumer.WithConsumeMessageBatchMaxSize(5), // 每次消费的最大消息数
consumer.WithConsumeGoroutineNums(20),      // 消费线程数
consumer.WithPullBatchSize(30),     // 每次拉取的消息数量

消息重试

若Consumer消费某条消息失败，则RocketMQ会在重试间隔时间后，将消息重新投递给Consumer消费，若达到最大重试次数后消息还没有成功被消费，则消息将被投递至死信队列

消息重试只针对集群消费模式生效；广播消费模式不提供失败重试特性，即消费失败后，失败消息不再重试，继续消费新的消息

  • 最大重试次数：消息消费失败后，可被重复投递的最大次数。

    consumer.WithMaxReconsumeTimes(5),

  • 重试间隔：消息消费失败后再次被投递给Consumer消费的间隔时间，只在顺序消费中起作用。

CODE

Simple

producer

package main

import (
    "context"
    "fmt"
    "github.com/apache/rocketmq-client-go/v2"
    "github.com/apache/rocketmq-client-go/v2/primitive"
    "github.com/apache/rocketmq-client-go/v2/producer"
    "os"
)

// Package main implements a simple producer to send message.
func main() {
    topic := "test-simple-topic"
    p, _ := rocketmq.NewProducer(
       producer.WithNameServer([]string{"127.0.0.1:9876"}),
       producer.WithRetry(2),
    )
    err := p.Start()
    if err != nil {
       fmt.Printf("start producer error: %s", err.Error())
       os.Exit(1)
    }

    tags := []string{"TagA", "TagB", "TagC"}
    for i := 0; i < 3; i++ {
       tag := tags[i%3]
       msg := primitive.NewMessage(topic, []byte("Hello RocketMQ Go Client!"))
       msg.WithTag(tag)

       res, err := p.SendSync(context.Background(), msg)
       if err != nil {
          fmt.Printf("send message error: %s\n", err)
       } else {
          fmt.Printf("send message success: result=%s\n", res.String())
       }
    }

    select {}
}

consumer

package main

import (
    "context"
    "fmt"
    "github.com/apache/rocketmq-client-go/v2"
    "github.com/apache/rocketmq-client-go/v2/consumer"
    "github.com/apache/rocketmq-client-go/v2/primitive"
    "os"
    "time"
)

func main() {
    topic := "test-simple-topic"
    c, _ := rocketmq.NewPushConsumer(
       consumer.WithGroupName("testSimpleGroup"),
       consumer.WithNameServer([]string{"127.0.0.1:9876"}),
    )
    selector := consumer.MessageSelector{
       Type:       consumer.TAG,
       Expression: "TagA || TagC",
    }
    err := c.Subscribe(
       topic,
       selector,
       func(ctx context.Context, msgs ...*primitive.MessageExt) (consumer.ConsumeResult, error) {
          fmt.Printf("subscribe callback: %v \n", msgs)
          return consumer.ConsumeSuccess, nil
       },
    )
    if err != nil {
       fmt.Println(err.Error())
    }
    err = c.Start()
    if err != nil {
       fmt.Println(err.Error())
       os.Exit(-1)
    }
    time.Sleep(time.Hour)
    err = c.Shutdown()
    if err != nil {
       fmt.Printf("shutdown Consumer error: %s", err.Error())
    }
}

Order

producer

package main

import (
    "context"
    "fmt"
    "github.com/apache/rocketmq-client-go/v2"
    "github.com/apache/rocketmq-client-go/v2/primitive"
    "github.com/apache/rocketmq-client-go/v2/producer"
    "strconv"
)

// Package main implements a simple producer to send message.
func main() {
    topic := "test-order-topic"
    p, err := rocketmq.NewProducer(
       producer.WithNameServer([]string{"127.0.0.1:9876"}),
       producer.WithRetry(2),
       producer.WithQueueSelector(producer.NewHashQueueSelector()),
    )
    if err != nil {
       panic(err)
    }

    err = p.Start()
    if err != nil {
       panic(err)
    }

    for i := 0; i < 3; i++ {
       orderId := strconv.Itoa(i)
       for j := 0; j < 5; j++ {
          msg := &primitive.Message{
             Topic: topic,
             Body:  []byte("Ordered Message Step -> " + strconv.Itoa(j)),
          }
          msg.WithShardingKey(orderId)
          res, err := p.SendSync(context.Background(), msg)
          if err != nil {
             fmt.Errorf("send message success: result=%s\n", res.String())
             continue
          }

          fmt.Printf("send message success: result=%s\n", res.String())
       }
    }

    // close producer
    err = p.Shutdown()
    if err != nil {
       fmt.Printf("shutdown producer error: %s\n", err.Error())
    }
}

consumer

package main

import (
    "context"
    "fmt"
    "github.com/apache/rocketmq-client-go/v2"
    "github.com/apache/rocketmq-client-go/v2/consumer"
    "github.com/apache/rocketmq-client-go/v2/primitive"
    "github.com/sirupsen/logrus"
    "os"
    "os/signal"
)

func main() {
    topic := "test-order-topic"
    c, err := rocketmq.NewPushConsumer(
       consumer.WithGroupName("testGroup"),
       consumer.WithNameServer([]string{"127.0.0.1:9876"}),
       consumer.WithConsumerOrder(true),
       consumer.WithConsumerModel(consumer.Clustering),
    )
    if err != nil {
       panic(err)
    }
    err = c.Subscribe(topic, consumer.MessageSelector{}, func(ctx context.Context,
       msgs ...*primitive.MessageExt) (consumer.ConsumeResult, error) {
       orderlyCtx, _ := primitive.GetOrderlyCtx(ctx)
       fmt.Printf("orderly context: %v\n", orderlyCtx)
       for i := range msgs {
          logrus.Infof("Receive message: %s, orderId: %s", msgs[i].Body, msgs[i].GetShardingKey())
       }
       return consumer.ConsumeSuccess, nil
    })
    if err != nil {
       panic(err)
    }
    // Note: start after subscribe
    err = c.Start()
    if err != nil {
       panic(err)
    }
    logrus.Info("Consumer Started.")
    stop := make(chan os.Signal)
    signal.Notify(stop, os.Interrupt, os.Kill)
    select {
    case <-stop:
       err = c.Shutdown()
       if err != nil {
          fmt.Printf("shutdown Consumer error: %s", err.Error())
       }
    }
}

Brocast

producer

package main

import (
    "context"
    "fmt"
    "github.com/apache/rocketmq-client-go/v2"
    "github.com/apache/rocketmq-client-go/v2/primitive"
    "github.com/apache/rocketmq-client-go/v2/producer"
    "os"
)

// Package main implements a simple producer to send message.
func main() {
    topic := "test-simple-topic"
    p, _ := rocketmq.NewProducer(
       producer.WithNameServer([]string{"127.0.0.1:9876"}),
       producer.WithRetry(2),
    )
    err := p.Start()
    if err != nil {
       fmt.Printf("start producer error: %s", err.Error())
       os.Exit(1)
    }

    tags := []string{"TagA", "TagB", "TagC"}
    for i := 0; i < 3; i++ {
       tag := tags[i%3]
       msg := primitive.NewMessage(topic, []byte("Hello RocketMQ Go Client!"))
       msg.WithTag(tag)

       res, err := p.SendSync(context.Background(), msg)
       if err != nil {
          fmt.Printf("send message error: %s\n", err)
       } else {
          fmt.Printf("send message success: result=%s\n", res.String())
       }
    }

    select {}
    //err = p.Shutdown()
    //if err != nil {
    // fmt.Printf("shutdown producer error: %s", err.Error())
    //}
}

consumer

package main

import (
    "context"
    "fmt"
    "github.com/apache/rocketmq-client-go/v2"
    "github.com/apache/rocketmq-client-go/v2/consumer"
    "github.com/apache/rocketmq-client-go/v2/primitive"
    "os"
    "time"
)

func main() {
    topic := "test-simple-topic"
    c, _ := rocketmq.NewPushConsumer(
       consumer.WithGroupName("testGroup2"),
       consumer.WithNameServer([]string{"127.0.0.1:9876"}),
       consumer.WithConsumerModel(consumer.BroadCasting),
    )
    err := c.Subscribe(
       topic,
       consumer.MessageSelector{},
       func(ctx context.Context, msgs ...*primitive.MessageExt) (consumer.ConsumeResult, error) {
          for _, msg := range msgs {
             fmt.Printf("Subscribe callback:%v\n", msg)
          }
          return consumer.ConsumeSuccess, nil
       },
    )
    if err != nil {
       fmt.Println(err.Error())
    }
    err = c.Start()
    if err != nil {
       fmt.Println(err.Error())
       os.Exit(-1)
    }
    time.Sleep(time.Hour)
    err = c.Shutdown()
    if err != nil {
       fmt.Printf("shutdown Consumer error: %s", err.Error())
    }
}

Batch

producer

package main

import (
    "context"
    "fmt"
    "github.com/apache/rocketmq-client-go/v2"
    "github.com/apache/rocketmq-client-go/v2/primitive"
    "github.com/apache/rocketmq-client-go/v2/producer"
    "os"
    "strconv"
)

func main() {
    topic := "test-batch-topic"
    p, err := rocketmq.NewProducer(
       producer.WithNameServer([]string{"127.0.0.1:9876"}),
       producer.WithRetry(2),
    )
    err = p.Start()
    if err != nil {
       fmt.Printf("start producer error: %s", err.Error())
       os.Exit(1)
    }
    var msgs []*primitive.Message
    for i := 0; i < 1000; i++ {
       msgs = append(msgs, primitive.NewMessage(topic, []byte("Hello RocketMQ Go Client! num: "+strconv.Itoa(i))))
    }
    res, err := p.SendSync(context.Background(), msgs...)
    if err != nil {
       fmt.Printf("send message error: %s\n", err)
    } else {
       fmt.Printf("send message success: result=%s\n", res.String())
    }
    err = p.Shutdown()
    if err != nil {
       fmt.Printf("shutdown producer error: %s", err.Error())
    }
}

consumer

package main

import (
    "context"
    "fmt"
    "github.com/apache/rocketmq-client-go/v2"
    "github.com/apache/rocketmq-client-go/v2/consumer"
    "github.com/apache/rocketmq-client-go/v2/primitive"
    "os"
    "time"
)

func main() {
    topic := "test-batch-topic"
    c, err := rocketmq.NewPushConsumer(
       consumer.WithGroupName("testBatchGroup"),
       consumer.WithNameServer([]string{"127.0.0.1:9876"}),
       consumer.WithConsumeMessageBatchMaxSize(2), // 设置批量消费，每次最多拉取 5 条消息
       consumer.WithConsumerModel(consumer.Clustering),
    )
    err = c.Subscribe(topic,
       consumer.MessageSelector{},
       func(ctx context.Context, msgs ...*primitive.MessageExt) (consumer.ConsumeResult, error) {
          fmt.Printf("收到批量消息: 本次共%d条\n", len(msgs))
          for _, msg := range msgs {
             fmt.Printf("Message ID: %s, Body: %s\n", msg.MsgId, string(msg.Body))
          }
          return consumer.ConsumeSuccess, nil
       },
    )
    if err != nil {
       fmt.Println(err.Error())
    }
    err = c.Start()
    if err != nil {
       fmt.Println(err.Error())
       os.Exit(-1)
    }
    time.Sleep(time.Hour)
    err = c.Shutdown()
    if err != nil {
       fmt.Printf("Shutdown Consumer error: %s", err.Error())
    }
}

Sync

producer

package main

import (
    "context"
    "fmt"
    "os"
    "sync"

    "github.com/apache/rocketmq-client-go/v2"
    "github.com/apache/rocketmq-client-go/v2/primitive"
    "github.com/apache/rocketmq-client-go/v2/producer"
)

// Package main implements a async producer to send message.
func main() {
    topic := "test-async-topic"
    p, _ := rocketmq.NewProducer(
       producer.WithNameServer([]string{"127.0.0.1:9876"}),
       producer.WithRetry(2))

    err := p.Start()
    if err != nil {
       fmt.Printf("start producer error: %s", err.Error())
       os.Exit(1)
    }
    var wg sync.WaitGroup
    for i := 0; i < 10; i++ {
       wg.Add(1)
       // async send
       err := p.SendAsync(context.Background(),
          func(ctx context.Context, result *primitive.SendResult, e error) {
             if e != nil {
                fmt.Printf("receive message error: %s\n", err)
             } else {
                fmt.Printf("send message success: result=%s\n", result.String())
             }
             wg.Done()
          },
          primitive.NewMessage(topic, []byte(fmt.Sprintf("Hello RocketMQ Go Client:%d", i))),
       )
       if err != nil {
          fmt.Printf("send message error: %s\n", err)
       }

       // send one way
       // p.SendOneWay(context.Background(), primitive.NewMessage(topic, []byte("Hello RocketMQ Go Client!")))
    }
    wg.Wait()
    err = p.Shutdown()
    if err != nil {
       fmt.Printf("shutdown producer error: %s", err.Error())
    }

    select {}
}

consumer

package main

import (
    "context"
    "fmt"
    "os"

    "github.com/apache/rocketmq-client-go/v2"
    "github.com/apache/rocketmq-client-go/v2/consumer"
    "github.com/apache/rocketmq-client-go/v2/primitive"
)

func main() {

    topic := "test-async-topic"

    sig := make(chan os.Signal)
    c, _ := rocketmq.NewPushConsumer(
       consumer.WithGroupName("testGroup"),
       consumer.WithNameServer([]string{"127.0.0.1:9876"}),
       consumer.WithConsumeMessageBatchMaxSize(5), // 每次消费的最大消息数
       consumer.WithConsumeGoroutineNums(20),      // 消费线程数
       consumer.WithPullBatchSize(30),             // 每次拉取的消息数量
    )
    err := c.Subscribe(topic, consumer.MessageSelector{},
       func(ctx context.Context, msgs ...*primitive.MessageExt) (consumer.ConsumeResult, error) {
          for i := range msgs {
             fmt.Printf("subscribe callback: %v \n", msgs[i])
          }
          return consumer.ConsumeSuccess, nil
       },
    )
    if err != nil {
       fmt.Println(err.Error())
    }
    // Note: start after subscribe
    err = c.Start()
    if err != nil {
       fmt.Println(err.Error())
       os.Exit(-1)
    }
    <-sig
    err = c.Shutdown()
    if err != nil {
       fmt.Printf("shutdown Consumer error: %s", err.Error())
    }
}

Delay

producer

package main

import (
    "context"
    "fmt"
    "github.com/apache/rocketmq-client-go/v2"
    "github.com/apache/rocketmq-client-go/v2/primitive"
    "github.com/apache/rocketmq-client-go/v2/producer"
    "os"
)

// Package main implements a simple producer to send message.
func main() {
    topic := "test-delay-topic"
    p, err := rocketmq.NewProducer(
       producer.WithNameServer([]string{"127.0.0.1:9876"}),
       producer.WithRetry(2),
    )
    err = p.Start()
    if err != nil {
       fmt.Printf("start producer error: %s", err.Error())
       os.Exit(1)
    }
    for i := 0; i < 10; i++ {
       msg := primitive.NewMessage(topic, []byte(fmt.Sprintf("Hello RocketMQ Go Client:%v", i)))
       msg.WithDelayTimeLevel(3)
       res, err := p.SendSync(context.Background(), msg)
       if err != nil {
          fmt.Printf("send message error: %s\n", err)
       } else {
          fmt.Printf("send message success: result=%s\n", res.String())
       }
    }
    err = p.Shutdown()
    if err != nil {
       fmt.Printf("shutdown producer error: %s", err.Error())
    }
}

consumer

package main

import (
    "context"
    "fmt"
    "github.com/apache/rocketmq-client-go/v2"
    "github.com/apache/rocketmq-client-go/v2/consumer"
    "github.com/apache/rocketmq-client-go/v2/primitive"
    "os"
    "time"
)

func main() {
    topic := "test-delay-topic"
    c, err := rocketmq.NewPushConsumer(
       consumer.WithGroupName("testGroup"),
       consumer.WithNameServer([]string{"127.0.0.1:9876"}),
    )
    err = c.Subscribe(topic, consumer.MessageSelector{},
       func(ctx context.Context, msgs ...*primitive.MessageExt) (consumer.ConsumeResult, error) {
          for _, msg := range msgs {
             t := time.Now().UnixNano()/int64(time.Millisecond) - msg.BornTimestamp
             fmt.Printf("Receive message[msgId=%s] %d ms later\n", msg.MsgId, t)
          }
          return consumer.ConsumeSuccess, nil
       })
    if err != nil {
       fmt.Println(err.Error())
    }
    // Note: start after subscribe
    err = c.Start()
    if err != nil {
       fmt.Println(err.Error())
       os.Exit(-1)
    }
    time.Sleep(time.Hour)
    err = c.Shutdown()
    if err != nil {
       fmt.Printf("Shutdown Consumer error: %s", err.Error())
    }
}

Transaction

producer

package main

import (
    "context"
    "fmt"
    "os"
    "strconv"
    "sync"
    "sync/atomic"
    "time"

    "github.com/apache/rocketmq-client-go/v2"
    "github.com/apache/rocketmq-client-go/v2/primitive"
    "github.com/apache/rocketmq-client-go/v2/producer"
)

type DemoListener struct {
    localTrans       *sync.Map
    transactionIndex int32
}

func NewDemoListener() *DemoListener {
    return &DemoListener{
       localTrans: new(sync.Map),
    }
}

// - 制触发事务回查机制
// 虽然计算了不同的状态并存储在 localTrans 中，但返回 UnknowState 会让 RocketMQ 服务器后续通过 CheckLocalTransaction 来回查事务状态
func (dl *DemoListener) ExecuteLocalTransaction(msg *primitive.Message) primitive.LocalTransactionState {
    nextIndex := atomic.AddInt32(&dl.transactionIndex, 1)
    fmt.Printf("nextIndex: %v for transactionID: %v\n", nextIndex, msg.TransactionId)
    dl.localTrans.Store(msg.TransactionId, primitive.LocalTransactionState(primitive.UnknowState))
    return primitive.UnknowState
}

func (dl *DemoListener) CheckLocalTransaction(msg *primitive.MessageExt) primitive.LocalTransactionState {
    fmt.Printf("%v msg transactionID : %v\n", time.Now(), msg.TransactionId)
    v, existed := dl.localTrans.Load(msg.TransactionId)
    if !existed {
       fmt.Printf("unknown msg: %v, return Commit", msg)
       return primitive.RollbackMessageState
    }
    state := v.(primitive.LocalTransactionState)
    switch state {
    case 1:
       fmt.Printf("checkLocalTransaction COMMIT_MESSAGE: %v\n", msg)
       return primitive.CommitMessageState
    case 2:
       fmt.Printf("checkLocalTransaction ROLLBACK_MESSAGE: %v\n", msg)
       return primitive.RollbackMessageState
    case 3:
       fmt.Printf("checkLocalTransaction unknown: %v\n", msg)
       return primitive.UnknowState
    default:
       fmt.Printf("checkLocalTransaction default ROLLBACK_MESSAGE: %v\n", msg)
       return primitive.RollbackMessageState
    }
}

func main() {
    p, _ := rocketmq.NewTransactionProducer(
       NewDemoListener(),
       producer.WithNameServer([]string{"127.0.0.1:9876"}),
       producer.WithRetry(1),
       producer.WithGroupName("test_transaction_producer_group"),
    )
    err := p.Start()
    if err != nil {
       fmt.Printf("start producer error: %s\n", err.Error())
       os.Exit(1)
    }

    for i := 0; i < 3; i++ {
       res, err := p.SendMessageInTransaction(context.Background(),
          primitive.NewMessage("test_transaction_topic", []byte("Hello RocketMQ again "+strconv.Itoa(i))))

       if err != nil {
          fmt.Printf("send message error: %s\n", err)
       } else {
          fmt.Printf("send message success: result=%s\n", res.String())
       }
    }

    select {}
}

consumer

package main

import (
    "context"
    "fmt"
    "os"
    "time"

    "github.com/apache/rocketmq-client-go/v2"
    "github.com/apache/rocketmq-client-go/v2/consumer"
    "github.com/apache/rocketmq-client-go/v2/primitive"
)

func main() {
    // 创建一个消费者实例
    c, _ := rocketmq.NewPushConsumer(
       consumer.WithNameServer([]string{"127.0.0.1:9876"}),
       consumer.WithGroupName("transaction_consumer_group"),
    )

    // 订阅主题和消息处理函数
    err := c.Subscribe(
       "test_transaction_topic",
       consumer.MessageSelector{},
       func(ctx context.Context, msgs ...*primitive.MessageExt) (consumer.ConsumeResult, error) {
          for i := range msgs {
             fmt.Printf("接收到消息: id=%v, topic=%v, body=%v, transactionId=%v\n",
                msgs[i].MsgId,
                msgs[i].Topic,
                string(msgs[i].Body),
                msgs[i].TransactionId)
          }
          return consumer.ConsumeSuccess, nil
       })

    if err != nil {
       fmt.Printf("订阅主题失败: %s\n", err.Error())
       os.Exit(1)
    }

    // 启动消费者
    err = c.Start()
    if err != nil {
       fmt.Printf("启动消费者失败: %s\n", err.Error())
       os.Exit(1)
    }

    // 等待消费
    time.Sleep(time.Hour)

    // 关闭消费者
    err = c.Shutdown()
    if err != nil {
       fmt.Printf("关闭消费者失败: %s\n", err.Error())
       os.Exit(1)
    }
}
avatar
文章
阅读
粉丝