分片原理

不使用分片

extends $\color{red}BaseSimpleJob$

使用分片

extends $\color{red}BaseShardingJob$

举例：

9个分片 shardingTotalCount = 9

1台机器([0,1,2,3,4,5,6,7,8]) 9个线程

2台机器([0,1,2,3,8] [4,5,6,7]) 5,4个线程

3台机器([0,1,2] [3,4,5] [6,7,8]) 3,3,3个线程

hash和9进行求余进入不同的线程执行数据

测试结果

2020-01-08 14:49:02.483 INFO 13812 --- [utoReceiveJob-3] 任务名称：AutoReceiveJob，分片信息，index=2，total=9

2020-01-08 14:49:02.483 INFO 13812 --- [utoReceiveJob-7] 任务名称：AutoReceiveJob，分片信息，index=6，total=9

2020-01-08 14:49:02.483 INFO 13812 --- [utoReceiveJob-2] 任务名称：AutoReceiveJob，分片信息，index=1，total=9

2020-01-08 14:49:02.483 INFO 13812 --- [utoReceiveJob-6] 任务名称：AutoReceiveJob，分片信息，index=5，total=9

2020-01-08 14:49:02.483 INFO 13812 --- [utoReceiveJob-1] 任务名称：AutoReceiveJob，分片信息，index=0，total=9

2020-01-08 14:49:02.483 INFO 13812 --- [utoReceiveJob-4] 任务名称：AutoReceiveJob，分片信息，index=3，total=9

2020-01-08 14:49:02.483 INFO 13812 --- [utoReceiveJob-5] 任务名称：AutoReceiveJob，分片信息，index=4，total=9

2020-01-08 14:49:02.483 INFO 13812 --- [utoReceiveJob-8] 任务名称：AutoReceiveJob，分片信息，index=7，total=9

2020-01-08 14:49:02.483 INFO 13812 --- [utoReceiveJob-9] 任务名称：AutoReceiveJob，分片信息，index=8，total=9

2020-01-08 14:49:02.484 INFO 13812 --- [utoReceiveJob-3] 任务名称：AutoReceiveJob，分片项：2，分片结果：[22, 4, 16]

2020-01-08 14:49:02.484 INFO 13812 --- [utoReceiveJob-1] 任务名称：AutoReceiveJob，分片项：0，分片结果：[2, 14, 20]

2020-01-08 14:49:02.484 INFO 13812 --- [utoReceiveJob-9] 任务名称：AutoReceiveJob，分片项：8，分片结果：[1, 13]

2020-01-08 14:49:02.484 INFO 13812 --- [utoReceiveJob-2] 任务名称：AutoReceiveJob，分片项：1，分片结果：[3, 15, 21]

2020-01-08 14:49:02.484 INFO 13812 --- [utoReceiveJob-4] 任务名称：AutoReceiveJob，分片项：3，分片结果：[23, 5, 17]

2020-01-08 14:49:02.484 INFO 13812 --- [utoReceiveJob-7] 任务名称：AutoReceiveJob，分片项：6，分片结果：[11, 8]

2020-01-08 14:49:02.484 INFO 13812 --- [utoReceiveJob-8] 任务名称：AutoReceiveJob，分片项：7，分片结果：[12, 9]

2020-01-08 14:49:02.484 INFO 13812 --- [utoReceiveJob-6] 任务名称：AutoReceiveJob，分片项：5，分片结果：[25, 7, 19, 10]

2020-01-08 14:49:02.484 INFO 13812 --- [utoReceiveJob-5] 任务名称：AutoReceiveJob，分片项：4，分片结果：[24, 6, 18]

maven 坐标

<dependency>
   <groupId>com.dangdang</groupId>
   <artifactId>elastic-job-lite-core</artifactId>
   <version>${elastic-job.version}</version>
</dependency>
      <dependency>
   <groupId>com.dangdang</groupId>
   <artifactId>elastic-job-lite-spring</artifactId>
   <version>${elastic-job.version}</version>
</dependency>

简单job

@Slf4j
public abstract class BaseSimpleJob implements SimpleJob {
 
    @Override
    public void execute(ShardingContext shardingContext) {
        // 任务开始时间
        long start = System.currentTimeMillis();
        try {
            log.info("{} start......", shardingContext.getJobName());
            // 执行任务
            this.doRun(shardingContext);
        } catch (Throwable e) {
            log.error("{} error......", shardingContext.getJobName(), e);
        } finally {
            // 任务结束时间
            long end = System.currentTimeMillis();
            log.info("{} end...... 用时：{}毫秒", shardingContext.getJobName(), end - start);
        }
    }
 
    /**
     * 运行服务
     *
     * @param shardingContext
     */
    public abstract void doRun(ShardingContext shardingContext);
}

简单job 用法

@Slf4j
@Component
@ElasticJobConf(name = "MonitorJob", cron = "0 0 9 * * ?", description = "库存对账")
public class MonitorJob extends BaseSimpleJob {
 
    @Autowired
    private MonitorBizService monitorBizService;
 
    @Override
    public void doRun(ShardingContext shardingContext) {
        monitorBizService.monitor();
    }
}

分片job

@Slf4j
public abstract class BaseShardingJob<T> extends BaseSimpleJob {
 
    @Override
    public void doRun(ShardingContext shardingContext) {
        Set<T> shardingDataList = this.listShardingData(shardingContext);
        log.info("分片信息：{}，分片结果：{}", shardingContext, shardingDataList);
        if (CollUtil.isEmpty(shardingDataList)) {
            return;
        }
 
        shardingDataList.forEach(data -> {
            try {
                // 执行分片任务
                this.doRun(data);
            } catch (Exception e) {
                // 分片执行失败后其他可以正常执行
                log.error("任务信息：{}，分片参数：{}，执行失败", shardingContext, data, e);
            }
        });
    }
 
    private Set<T> listShardingData(ShardingContext shardingContext) {
        // 获取所有可运行数据
        Set<T> processDataList = this.listProcessData();
        if (CollectionUtil.isEmpty(processDataList)) {
            return Collections.emptySet();
        }
 
        // 分片数据
        Set<T> shardingDataList = new HashSet<>();
        processDataList.forEach(data -> {
            int jobHashCode = Objects.hash(data);
            if (shardingContext.getShardingItem() == (jobHashCode % shardingContext.getShardingTotalCount())) {
                shardingDataList.add(data);
            }
        });
        return shardingDataList;
    }
 
    /**
     * 每个分片业务逻辑处理方法
     *
     * @param t
     */
    public abstract void doRun(T t);
 
    /**
     * 获取所有分片可运行的数据
     *
     * @return 所有分片可运行的数据
     */
    public abstract Set<T> listProcessData();
 
}

分片job 用法

@Slf4j
@Component
@ElasticJobConf(name = "QtyPriceMonthJob", cron = "0 0 4 * * ?", shardingTotalCount = 2, description = "库存进入存月报表")
public class QtyPriceMonthJob extends BaseShardingJob<Long> {
 
    @Autowired
    private QtyPriceMonthBizService qtyPriceMonthBizService;
     
    @Autowired
    private TrsQtyService trsQtyService;
 
    @Override
    public void doRun(Long warehouseId) {
        QtyPriceMonthDto dto = new QtyPriceMonthDto();
        dto.setWarehouseId(warehouseId);
        dto.setPriceDate(DateUtil.lastMonth());
        qtyPriceMonthBizService.statsQtyPriceMonth(dto);
    }
 
    @Override
    public Set<Long> listProcessData() {
        List<TrsQty> trsQtyList = trsQtyService.listPriceWarehouseId();
        if (CollUtil.isEmpty(trsQtyList)) {
            return Collections.emptySet();
        }
        return trsQtyList.stream().map(TrsQty::getWarehouseId).collect(Collectors.toSet());
    }
}

job.properties

#elastic job listener
elastic.job.listener=com.xxx.job.JobListener

 
#elastic job exception
elastic.job.jobExceptionHandler=com.xxx.job.JobFailHandler

Job Listener

@Slf4j
public class JobListener implements ElasticJobListener {
 
    @Override
    public void beforeJobExecuted(ShardingContexts shardingContexts) {
        MDC.put(Constant.REQUEST_ID, UUID.randomUUID().toString(true));
        String dateTime = DateUtil.formatDateTime(new Date());
        log.debug("{} 定时任务【{}】开始执行==== {}", dateTime, shardingContexts.getJobName(), JsonUtils.toJson(shardingContexts));
    }
 
    @Override
    public void afterJobExecuted(ShardingContexts shardingContexts) {
        String dateTime = DateUtil.formatDateTime(new Date());
        log.debug("{} 定时任务【{}】执行结束==== {}", dateTime, shardingContexts.getJobName(), JsonUtils.toJson(shardingContexts));
        MDC.clear();
    }
 
}

Job Exception

@Slf4j
public class JobFailHandler implements JobExceptionHandler {
 
    @Override
    public void handleException(String jobName, Throwable throwable) {
        String dateTime = DateUtil.formatDateTime(new Date());
        log.warn("{} 定时任务【{}】执行失败==== {}", dateTime, jobName, throwable.getMessage());
    }
}