单线程读取 多线程执行

代码

@Service
@Slf4j
public abstract class CsvJobService {

    /**
     * deque的大小
     */
    public static final int DEQUE_SIZE = 10000;

    /**
     * 线程数量
     */
    public static final int THREAD_SIZE = 4;

    /**
     * 数据单次入库条数
     */
    public static final int SINGLE_SIZE = 1000;

    /**
     * 导入cvs文件
     *
     * @param filePath 文件路径
     */
    public void importCVSFile(String filePath) {
        //阻塞队列,采用独立的写锁putLock和读锁takeLock,高并发读写行性能比ArrayBlockingQueue好
        BlockingDeque<String> deque = new LinkedBlockingDeque<>(DEQUE_SIZE);
        //开始时间
        long startTime = System.currentTimeMillis();

        //生产者 读取cvs文件，数据放入deque
        ExecutorService product = Executors.newSingleThreadExecutor();
        product.execute(() -> {
            try {
                BufferedReader reader = Files.newBufferedReader(Paths.get(filePath));
                for (int i = 0; ; i++) {
                    String line = reader.readLine();
                    if (line == null) {
                        log.info("------生产者完成：总数据：" + i + "条");
                        break;
                    }
                    deque.put(line);
                }
            } catch (Exception e) {
                log.error("------读取文件异常", e);
            }
        });

        CountDownLatch countDownLatch = new CountDownLatch(THREAD_SIZE);
        //固定大小的子线程池
        ExecutorService executorService = Executors.newFixedThreadPool(THREAD_SIZE);
        //防止阻塞，多线程哪个执行完先返回哪个
        ExecutorCompletionService<Integer> executorCompletionService = new ExecutorCompletionService<>(executorService);
        for (int i = 0; i < THREAD_SIZE; i++) {
            //循环开启线消费者线程
            executorCompletionService.submit(new Customer(deque, countDownLatch));
        }

        try {
            //等待子线程执行完
            countDownLatch.await();
        } catch (InterruptedException e) {
            e.printStackTrace();
        }
        int jobSum = 0;
        for (int i = 0; i < THREAD_SIZE; i++) {
            try {
                //子线程结果
                Integer temp = executorCompletionService.take().get();
                jobSum += temp;
            } catch (InterruptedException | ExecutionException e) {
                e.printStackTrace();
            }
        }

        product.shutdown();
        executorService.shutdown();
        long endTime = System.currentTimeMillis();
        log.info("导入任务完成,总数据：{}条,用时：{}秒", jobSum, (endTime - startTime) / 1000);
    }

    /**
     * 消费者
     */
    class Customer implements Callable<Integer> {

        private final BlockingDeque<String> deque;
        private final CountDownLatch countDownLatch;

        public Customer(BlockingDeque<String> deque, CountDownLatch countDownLatch) {
            this.deque = deque;
            this.countDownLatch = countDownLatch;
        }

        @Override
        public Integer call() throws Exception {
            int sum = 0;
            List<String> list = new LinkedList<>();
            //循环消费队列中的数据
            for (; ; ) {
                String poll = deque.poll(1, TimeUnit.SECONDS);
                if (poll == null) {
                    sum += list.size();
                    saveJob(list);
                    list.clear();
                    break;
                }
                list.add(poll);
                //当list大小>=设定的单次数据量，执行saveJob导入
                if (list.size() >= SINGLE_SIZE) {
                    sum += list.size();
                    saveJob(list);
                    list.clear();
                }
            }
            countDownLatch.countDown();
            return sum;
        }
    }

    /**
     * 数据处理方法
     *
     * @param list s
     */
    public abstract void saveJob(List<String> list);

}