背景

最近在做一个告警任务，考虑的效率问题，方案上将一部分数据进行预热。但同时由于这批数据比较大，因此再预热的同时采用了多线程的方式来执行，提升任务处理的效率。

这里的切入点是在项目启动后去执行预热的操作，就实现了CommandLineRunner 接口，在整个项目启动完毕后自动执行预热操作。其他的操作就剩下将需要预热的数据查询出来写入redis中。预热后的数据存在缓存中，在业务数据被修改后，同时需要在业务代码中对该部分数据做同步更新操作，保证DB与缓存中的数据一致性。


@Slf4j
@Component
public class InitAlarmConditionCache implements CommandLineRunner {

    @Resource
    private StringRedisTemplate stringRedisTemplate;

    @Resource
    private AlarmConditionGateway alarmConditionGateway;


    // 将总数分为parts等份
    private static final int parts = 10;

    @Override
    public void run(String... args) {
        log.info(">>>>>>>>>>>>>>> InitAlarmConditionCache start");
        // 1.获取到数据总数
        List<Long> equipmentIdList = alarmConditionGateway.selectEnableAlarmCondition();
        int total = equipmentIdList.size();
        if (total == 0) {
            return;
        }

        log.info(">>>>>>>>>>>>>>> InitAlarmConditionCache size {}", equipmentIdList.size());

        List<InitAlarmConditionProcessor> tasks = new ArrayList<>();
        // 计算每个等份的大小
        int partitionSize = total / parts;

        // 计算每个等份的开始位置和结束位置
        for (int i = 0; i < parts; i++) {
            int start = i * partitionSize;
            int end = (i + 1) * partitionSize - 1;
            if (i == parts - 1) {
                end = total - 1;
            }
            tasks.add(new InitAlarmConditionProcessor(stringRedisTemplate, alarmConditionGateway, start, end, equipmentIdList));
        }

        // 启动多个线程处理每个等份的数据
        List<Thread> threads = new ArrayList<>();
        for (InitAlarmConditionProcessor task : tasks) {
            Thread thread = new Thread(task);
            thread.start();
            threads.add(thread);
        }

        // 等待所有线程执行完毕
        for (Thread thread : threads) {
            try {
                thread.join();
            } catch (InterruptedException e) {
                log.info(">>>>>>>>>>>>>>> InitAlarmConditionCache exception:", e);
                e.printStackTrace();
            }
        }

        log.info(">>>>>>>>>>>>>>> InitAlarmConditionCache end");
    }

}


public class InitAlarmConditionProcessor implements Runnable {
    private final StringRedisTemplate stringRedisTemplate;

    private final AlarmConditionGateway alarmConditionGateway;

    private final int start;

    private final int end;

    private final List<Long> equipmentIdList;


    public InitAlarmConditionProcessor(StringRedisTemplate stringRedisTemplate, AlarmConditionGateway alarmConditionGateway, int start, int end, List<Long> equipmentIdList) {
        this.stringRedisTemplate = stringRedisTemplate;
        this.alarmConditionGateway = alarmConditionGateway;
        this.start = start;
        this.end = end;
        this.equipmentIdList = equipmentIdList;
    }

    @Override
    public void run() {
        // 处理该等份的数据
        for (int i = start; i <= end; i++) {
            List<AlarmConditionModel> alarmConditionModelList = alarmConditionGateway.selectAlarmConditionByEquipmentId(equipmentIdList.get(i));
            if (!CollectionUtils.isEmpty(alarmConditionModelList)) {
                Integer equipmentNumber = alarmConditionModelList.get(0).getEquipmentNumber();
                String alarmConditionKey = RedisCacheKeyEnum.NEW_ALARM_CONDITION_CACHE_KEY.getKey().concat(String.valueOf(equipmentNumber));
                AlarmConditionCache conditionCache = new AlarmConditionCache();
                conditionCache.setAlarmConditionModelList(alarmConditionModelList);
                stringRedisTemplate.delete(alarmConditionKey);
                stringRedisTemplate.opsForValue().set(alarmConditionKey, JSONObject.toJSONString(conditionCache));
            }
        }
    }

}

缓存预热大批量数据--多线程操作

背景