写一个项目去并发处理DB任务

1. 应用的场景

1. 某公司互动娱乐事业部需要把他们的DB记录进行处理，经过一些列轮转最终反馈给客户

2. 业务特色

1. 自动扫描DB的任务
2. 需要优雅的关闭我们处理DB的任务
3. 高QPS，需要异步处理
4. 批次处理任务需要同时完成了才能进行下一步

3. 代码实战

/**
 * 消息实体
 */
@Data
public class Message {
    private String orderId;
}




/**
 * 永动任务的模版
 */
@Slf4j
public abstract class TemplateTask<T> {
    /**
     * 默认线程池大小
     */
    private int poolSize = 5;

    /**
     * 默认数据拆分大小
     */
    private int splitSize = 5;

    /**
     * 接收jvm关闭信号，实现优雅停机
     */
    protected volatile boolean terminal = false;

    public void setPoolSize(int poolSize) {
        this.poolSize = Math.min(10, poolSize);
    }
    public void setSplitSize(int splitSize) {
        this.splitSize = Math.min(10, splitSize);
    }

    /**
     * 获取数据
     */
    public abstract List<T> queryData();

    /**
     * 业务逻辑处理
     */
    public abstract void execute(T data);

    /**
     * 设置执行间隔
     */
    public abstract int inter();

    /**
     * 任务名称
     */
    public abstract String serviceName();

    /**
     * 程序执行入口
     */
    public void doExecute() {
        int i = 0;
        try {
            while (true) {
                log.info("{} 任务第 {} 次执行",serviceName(),i);
                // 获取数据
                List<T> datas = queryData();
                // 处理数据
                taskExecute(datas);
                log.info("{}任务第{}次执行结束",serviceName(),i);
                if (terminal) {
                    // 只有应用关闭，才会走到这里，用于实现优雅的下线
                    break;
                }
                // 设置线程的休息时间
                if (inter() > 0){
                    Thread.sleep(inter());
                }
                i++;
            }
        }catch (Exception e){
            throw new RuntimeException(e);
        }finally {
            // 回收线程池资源
            TaskProcessUtil.releaseExecutors(serviceName());
        }
    }

    /**
     *  优雅停机
     */
    public void terminal() {
        // 关机
        terminal = true;
        log.info("{} shut down",serviceName());
    }

    /**
     * 处理数据
     */
    private void doProcessData(List<T> datas, CountDownLatch latch) {
        try {
            for (T cat : datas) {
                execute(cat);
            }
        } catch (Exception e) {
            log.error(Arrays.toString(e.getStackTrace()));
        } finally {
            if (latch != null) {
                latch.countDown();
            }
        }
    }

    /**
     * 处理单个任务数据
     */
    private void taskExecute(List<T> data) {
        if (CollectionUtils.isEmpty(data)) {
            return;
        }
        // 最大是SPLIT_SIZE
        splitSize = Math.min(this.splitSize, data.size());
        // 拆分数据
        List<List<T>> splitData = Lists.partition(data,  splitSize);
        // 子线程同步锁声明
        final CountDownLatch latch = new CountDownLatch(splitData.size());

        // 并发处理拆分的数据，共用一个线程池
        for (final List<T> datas : splitData) {
            ExecutorService executorService = TaskProcessUtil.getOrInitExecutors(serviceName(), poolSize);
            executorService.submit(() -> { doProcessData(datas, latch); });
        }
        try {
            latch.await();
        } catch (Exception e) {
            log.error(Arrays.toString(e.getStackTrace()));
        }
    }
}




/**
 * 自定义的永动任务
 */
@Component
public class ConcreteTask extends TemplateTask<Message> {
    @Override
    public List<Message> queryData() {
        Message message = new Message();
        message.setOrderId("0001");
        List<Message> messages = Arrays.asList(message);
        return messages;
    }
    @Override
    public void execute(Message data) {
        System.out.println("每条信息的处理"+data.getOrderId());
    }
    @Override
    public int inter() {
        return 1000;
    }
    @Override
    public String serviceName() {
        return "ConcreteTask1";
    }
}



/**
 * 线程池工具类
 */
public class TaskProcessUtil {
    /**
     * 每个任务，都有自己单独的线程池
     */
    private static Map<String, ExecutorService> executorMap = new ConcurrentHashMap<>();

    private static ExecutorService init(String poolName, int poolSize) {
        return new ThreadPoolExecutor(poolSize, poolSize,
                0L, TimeUnit.MILLISECONDS,
                new LinkedBlockingQueue<Runnable>(),
                new ThreadFactoryBuilder().setNameFormat("Pool-" + poolName).setDaemon(false).build(),
                new ThreadPoolExecutor.CallerRunsPolicy());
    }

    /**
     * 获取线程池
     */
    public static ExecutorService getOrInitExecutors(String poolName,int poolSize) {
        ExecutorService executorService = executorMap.get(poolName);
        if (null == executorService) {
            synchronized (TaskProcessUtil.class) {
                executorService = executorMap.get(poolName);
                if (null == executorService) {
                    executorService = init(poolName, poolSize);
                    executorMap.put(poolName, executorService);
                }
            }
        }
        return executorService;
    }

    // 回收线程资源
    public static void releaseExecutors(String poolName) {
        ExecutorService executorService = executorMap.remove(poolName);
        if (executorService != null) {
            executorService.shutdown();
        }
    }
}





/**
 * 开启永动任务的门面
 */
@Component
public class TaskManager implements InitializingBean {

    @Autowired
    List<TemplateTask> templateList;

    /**
     *  初始化任务
     */
    public void initTasks() {
        for (TemplateTask childTask : templateList) {
            ExecutorService commonTaskExecutor = TaskProcessUtil.getOrInitExecutors("COMMON_TASK", 10);
            commonTaskExecutor.submit(childTask::doExecute);
        }
    }

    /**
     * 关闭任务
     */
    public void shutdownTasks() {
        if (!CollectionUtils.isEmpty(templateList)) {
            for (TemplateTask childTask : templateList) {
                childTask.terminal();
            }
        }
    }

    /**
     * 通过任务名字关闭任务
     */
    public void shutdownTaskByName(String serviceName) {
        Optional<TemplateTask> first = templateList.stream().filter(service -> service.serviceName().equals(serviceName)).findFirst();
        first.ifPresent(TemplateTask::terminal);
    }

    /**
     * Springboot容器启动时默认开启任务
     */
    @Override
    public void afterPropertiesSet() throws Exception {
        initTasks();
    }

}