1 machinery主要模块
Server：业务模块，生成具体任务，可根据业务逻辑中，按交互进行拆分;
Broker：存储具体序列化后的任务，machinery中目前支持到Redis、AMQP和SQS;
Worker：工作进程，负责消费者功能，处理具体的任务;
Backend：后端存储，用于存储任务执行状态的数据;
2 基本功能原理
AsyncTask 一般的异步任务
DelayTask 延迟异步任务
TaskChain 异步任务链
3 Backend适配MySQL
3.1 Backend接口
type Backend interface {
// Group related functions
InitGroup(groupUUID string, taskUUIDs []string) error
GroupCompleted(groupUUID string, groupTaskCount int) (bool, error)
GroupTaskStates(groupUUID string, groupTaskCount int) ([]*tasks.TaskState, error)
TriggerChord(groupUUID string) (bool, error)
// Setting / getting task state
SetStatePending(signature *tasks.Signature) error
SetStateReceived(signature *tasks.Signature) error
SetStateStarted(signature *tasks.Signature) error
SetStateRetry(signature *tasks.Signature) error
SetStateSuccess(signature *tasks.Signature, results []*tasks.TaskResult) error
SetStateFailure(signature *tasks.Signature, err string) error
GetState(taskUUID string) (*tasks.TaskState, error)
// Purging stored stored tasks states and group meta data
IsAMQP() bool
PurgeState(taskUUID string) error
PurgeGroupMeta(groupUUID string) error
}
InitGroup()，在创建一个Group任务
GroupCompleted()，检查一个Group中所有的任务是否都执行完毕
GroupTaskStates()，返回一个Group中，所有任务的状态
TriggerChord()，当Group中任务全部执行完毕后，触发Chrod任务
SetSatete系列接口，任务状态如下：
Pending，任务到达Broker
Received，任务从Broker中读取成功
Started，任务开始执行
Retry，任务需要重试
Success，任务执行成功
Failure，任务执行失败
PurgeState() 清除任务状态
PurgeGroupMeta() 清除任务组状态
3.2 数据库表设计
CREATE TABLE task_state (
id bigint unsigned NOT NULL AUTO_INCREMENT,
uuid varchar(255) NOT NULL DEFAULT '' COMMENT 'uuid',
name varchar(255) NOT NULL DEFAULT '' COMMENT '异步任务名称',
state varchar(255) NOT NULL DEFAULT '' COMMENT '异步任务状态',
results text COMMENT '异步任务执行结果',
error_info text COMMENT '错误',
created_at datetime(6) NOT NULL COMMENT '创建时间',
PRIMARY KEY (id),
UNIQUE INDEX idx_task_state_uuid (uuid)
) ENGINE=InnoDB CHARSET=utf8mb4 COMMENT='异步任务状态表';
CREATE TABLE group_metadata (
id bigint unsigned NOT NULL AUTO_INCREMENT,
group_uuid varchar(255) NOT NULL DEFAULT '' COMMENT '异步任务组uuid',
tasks_uuid text COMMENT '异步任务uuid',
chord_enabled tinyint(1) DEFAULT 0 COMMENT 'chord开关',
created_at datetime(6) NOT NULL COMMENT '创建时间',
PRIMARY KEY (id),
UNIQUE INDEX idx_group_metadata_group_uuid (group_uuid)
) ENGINE=InnoDB CHARSET=utf8mb4 COMMENT='异步任务组元数据表';
3.3 代码实现
type SqlConfig struct {
Cnf *config.Config
DB *gorm.DB
}
type Backend struct {
common.Backend
db *gorm.DB
}
// New creates Backend instance
func New(cnf *SqlConfig) iface.Backend {
return &Backend{
Backend: common.NewBackend(cnf.Cnf),
db: cnf.DB,
}
}
// InitGroup creates and saves a group metadata object
func (b *Backend) InitGroup(groupUUID string, taskUUIDs []string) error {
groupMeta := &GroupMetadata{
GroupUUID: groupUUID,
TaskUUIDS: strings.Join(taskUUIDs, taskUUIDSplitSep),
CreatedAt: time.Now().UTC(),
}
return b.db.Create(groupMeta).Error
}
// GroupCompleted returns true if all tasks in a group finished
func (b *Backend) GroupCompleted(groupUUID string, groupTaskCount int) (bool, error) {
mGroup, err := b.getGroup(groupUUID)
if err != nil {
return false, err
}
var c int64 = 0
taskUUIDS := strings.Split(mGroup.TaskUUIDS, taskUUIDSplitSep)
if err := b.db.Table(taskStateTableName).Where("uuid in ? AND state = ? or state = ?", taskUUIDS, tasks.StateSuccess, tasks.StateFailure).
Count(&c).Error; err != nil {
return false, err
}
return int64(groupTaskCount) == c, nil
}
// GroupTaskStates returns states of all tasks in the group
func (b *Backend) GroupTaskStates(groupUUID string, groupTaskCount int) ([]*tasks.TaskState, error) {
mGroup, err := b.getGroup(groupUUID)
if err != nil {
return nil, err
}
mTasks := make([]*TaskState, 0)
taskUUIDS := strings.Split(mGroup.TaskUUIDS, taskUUIDSplitSep)
if err := b.db.Table(taskStateTableName).Where("uuid in ?", taskUUIDS).Find(&mTasks).Error; err != nil {
return nil, err
}
taskStates, err := b.mTasks2TaskStates(mTasks...)
if err != nil {
return nil, err
}
return taskStates, nil
}
// TriggerChord flags chord as triggered in the backend storage to make sure
// chord is never triggerred multiple times. Returns a boolean flag to indicate
// whether the worker should trigger chord (true) or no if it has been triggered
// already (false)
func (b *Backend) TriggerChord(groupUUID string) (bool, error) {
result := b.db.Table(groupMetadataTableName).
Where("group_uuid = ? AND chord_triggered = false", groupUUID).
Update("chord_triggered", true)
if result.Error != nil {
return false, result.Error
}
if result.RowsAffected == 0 {
return false, nil
}
return true, nil
}
// SetStatePending updates task state to PENDING
func (b *Backend) SetStatePending(signature *tasks.Signature) error {
m := &TaskState{
UUID: signature.UUID,
Name: signature.Name,
State: tasks.StatePending,
CreatedAt: time.Now().UTC(),
}
return b.db.Create(m).Error
}
// SetStateReceived updates task state to RECEIVED
func (b *Backend) SetStateReceived(signature *tasks.Signature) error {
return b.updateState(signature.UUID, tasks.StateReceived)
}
// SetStateStarted updates task state to STARTED
func (b *Backend) SetStateStarted(signature *tasks.Signature) error {
return b.updateState(signature.UUID, tasks.StateStarted)
}
// SetStateRetry updates task state to RETRY
func (b *Backend) SetStateRetry(signature *tasks.Signature) error {
return b.updateState(signature.UUID, tasks.StateRetry)
}
// SetStateSuccess updates task state to SUCCESS
func (b *Backend) SetStateSuccess(signature *tasks.Signature, results []*tasks.TaskResult) error {
encoded, err := json.Marshal(results)
if err != nil {
return err
}
return b.db.Table(taskStateTableName).Where("uuid = ?", signature.UUID).
Updates(map[string]interface{}{"state": tasks.StateSuccess, "results": string(encoded)}).Error
}
// SetStateFailure updates task state to FAILURE
func (b *Backend) SetStateFailure(signature *tasks.Signature, err string) error {
return b.db.Table(taskStateTableName).Where("uuid = ?", signature.UUID).
Updates(map[string]interface{}{"state": tasks.StateFailure, "error": err}).Error
}
// GetState returns the latest task state
func (b *Backend) GetState(taskUUID string) (*tasks.TaskState, error) {
m := &TaskState{}
if err := b.db.Where("uuid = ?", taskUUID).First(m).Error; err != nil {
return nil, err
}
taskStates, err := b.mTasks2TaskStates(m)
if err != nil {
return nil, err
}
return taskStates[0], nil
}
// PurgeState deletes stored task state
func (b *Backend) PurgeState(taskUUID string) error {
return b.db.Where("uuid = ?", taskUUID).Delete(&TaskState{}).Error
}
// PurgeGroupMeta deletes stored group meta data
func (b *Backend) PurgeGroupMeta(groupUUID string) error {
return b.db.Where("group_uuid = ?", groupUUID).Delete(&GroupMetadata{}).Error
}
func (b *Backend) updateState(uuid string, state string) error {
return b.db.Table(taskStateTableName).Where("uuid = ?", uuid).
Updates(map[string]interface{}{"state": state}).Error
}
func (b *Backend) getGroup(groupUUID string) (*GroupMetadata, error) {
mGroup := &GroupMetadata{}
if err := b.db.Where("group_uuid = ?", groupUUID).First(mGroup).Error; err != nil {
return nil, err
}
return mGroup, nil
}
func (b *Backend) mTasks2TaskStates(mTasks ...*TaskState) ([]*tasks.TaskState, error) {
takeStates := make([]*tasks.TaskState, len(mTasks), len(mTasks))
for i, m := range mTasks {
results := make([]*tasks.TaskResult, 0)
if m.Results != "" {
decoder := json.NewDecoder(bytes.NewReader([]byte(m.Results)))
decoder.UseNumber()
if err := decoder.Decode(&results); err != nil {
return nil, err
}
}
takeStates[i] = &tasks.TaskState{
TaskUUID: m.UUID,
TaskName: m.Name,
State: m.State,
Results: results,
Error: m.Error,
CreatedAt: m.CreatedAt,
}
}
return takeStates, nil
}
