Mysql 批量更新

272 阅读1分钟

本文已参与「新人创作礼」活动,一起开启掘金创作之路。

事前准备

先往数据库中添加 1w 条数据

简单循环事务提交

/**
 * 循环操作,自动事务提交
 * 耗时 1066816 ms
 */
@Test
public void testUpdateV1() {
    List<MybatisTest> list = mapper.selectList(new QueryWrapper<>());
    long start = System.currentTimeMillis();
    for (MybatisTest test : list) {
        test.setTestStr("test_str1");
        mapper.updateById(test);
    }
    long end = System.currentTimeMillis();
    System.out.println("总计耗时:" + (end - start));
}

感觉耗时有点高,不知道是什么原因导致的

手动事务提交

/**
 * 循环操作,手动事务提交
 * 耗时 37869 ms
 */
@Test
public void testUpdateV2() {
    List<MybatisTest> list = mapper.selectList(new QueryWrapper<>());
    long start = System.currentTimeMillis();
    TransactionStatus transactionStatus = dataSourceTransactionManager.getTransaction(transactionDefinition);
    try {
        for (MybatisTest test : list) {
            test.setTestStr("test_str2");
            mapper.updateById(test);
        }
        dataSourceTransactionManager.commit(transactionStatus);
    } catch (Exception e) {
        dataSourceTransactionManager.rollback(transactionStatus);
    } finally {
        long end = System.currentTimeMillis();
        System.out.println("总计耗时:" + (end - start));
    }
}

不是每次更新都提交一次事务,所以效率会提高

批量提交

/**
 * 批量更新
 * 耗时: 5480 ms
 */
@Test
public void testUpdateV4() {
    List<MybatisTest> list = mapper.selectList(new QueryWrapper<>());
    List<MybatisTest> result = new ArrayList<>();
    long start = System.currentTimeMillis();
    for (MybatisTest mybatisTest : list) {
        mybatisTest.setTestStr("test_str1");
        result.add(mybatisTest);
    }
    mapper.updateBatchById(result);
    long end = System.currentTimeMillis();
    System.out.println("总耗时:" + (end - start));
}

多线程事务提交


    /**
     * 多线程操作,手动事务提交
     * 子线程:pool-1-thread-1耗时:9875 ms
     * 子线程:pool-1-thread-4耗时:9910 ms
     * 子线程:pool-1-thread-3耗时:10055 ms
     * 子线程:pool-1-thread-5耗时:10054 ms
     * 子线程:pool-1-thread-2耗时:10325 ms
     */
    @Test
    public void testUpdateV3() {
        List<MybatisTest> list = mapper.selectList(new QueryWrapper<>());
        // 线程数量
        final int threadCount = 5;
        // 每个线程处理的数据量
        final int dataPartionLength = (list.size() + threadCount - 1) / threadCount;
        // 创建多线程处理任务
        ExecutorService studentThreadPool = Executors.newFixedThreadPool(threadCount);
        CountDownLatch countDownLatch = new CountDownLatch(threadCount);
        for (int i = 0; i < threadCount; i++) {
            // 每个线程处理的数据
            List<MybatisTest> tmp = list.stream()
                    .skip((long) i * dataPartionLength).limit(dataPartionLength).collect(Collectors.toList());
            studentThreadPool.execute(() -> {
                testUpdate(tmp, countDownLatch);
            });
        }
        try {
            // 倒计时锁设置超时时间 30s
            countDownLatch.await(30, TimeUnit.SECONDS);
        } catch (Throwable e) {
            e.printStackTrace();
        }
        System.out.println("主线程完成");
    }

    private void testUpdate(List<MybatisTest> list, CountDownLatch countDownLatch) {
        TransactionStatus transactionStatus = dataSourceTransactionManager.getTransaction(transactionDefinition);
        long start = System.currentTimeMillis();
        try {
            list.forEach(s -> {
                s.setTestStr("test_str3");
                mapper.updateById(s);
            });
            dataSourceTransactionManager.commit(transactionStatus);
            countDownLatch.countDown();
        } catch (Throwable e) {
            e.printStackTrace();
            dataSourceTransactionManager.rollback(transactionStatus);
        } finally {
            long end = System.currentTimeMillis();
            System.out.println("子线程:" + Thread.currentThread().getName() + "耗时:" + (end - start));
        }
    }


基于 CountDownLatch 控制多线程事务提交

   /**
     * 多线程操作,手动控制事务
     * 子线程提交:pool-1-thread-1耗时:8240 ms
     * 子线程提交:pool-1-thread-3耗时:8295 ms
     * 子线程提交:pool-1-thread-2耗时:8334 ms
     * 子线程提交:pool-1-thread-5耗时:8122 ms
     * 子线程提交:pool-1-thread-4耗时:8181 ms
     */
    @Test
    public void testUpdateV5() {
        List<MybatisTest> list = mapper.selectList(new QueryWrapper<>());
        // 线程数量
        final int threadCount = 5;
        // 每个线程处理的数据量
        final int dataPartionLength = (list.size() + threadCount - 1) / threadCount;
        // 创建多线程处理任务
        ExecutorService studentThreadPool = Executors.newFixedThreadPool(threadCount);
        // 用于计算子线程提交数量
        CountDownLatch countDownLatch = new CountDownLatch(threadCount);
        // 用于判断主线程是否提交
        CountDownLatch mainLatch = new CountDownLatch(1);
        // 用于判断子线程任务是否有错误
        AtomicBoolean flag = new AtomicBoolean();
        for (int i = 0; i < threadCount; i++) {
            // 每个线程处理的数据
            List<MybatisTest> tmp = list.stream()
                    .skip((long) i * dataPartionLength).limit(dataPartionLength).collect(Collectors.toList());
            studentThreadPool.execute(() -> {
                testUpdate(tmp, countDownLatch, mainLatch, flag);
            });
        }
        try {
            // 倒计时锁设置超时时间 30s
            boolean await = countDownLatch.await(30, TimeUnit.SECONDS);
            if (!await) {
                System.out.println("执行超时!");
                flag.set(true);
            }
        } catch (Throwable e) {
            e.printStackTrace();
            flag.set(true);
        }
        mainLatch.countDown();
        studentThreadPool.shutdown();
        System.out.println("主线程完成");
        System.out.println(flag);
    }

    private void testUpdate(List<MybatisTest> list, CountDownLatch countDownLatch, CountDownLatch mainLatch, AtomicBoolean flag) {
        TransactionStatus transactionStatus = dataSourceTransactionManager.getTransaction(transactionDefinition);
        long start = System.currentTimeMillis();
        try {
            list.forEach(s -> {
                if (s.getId() == 5000L) {
                    throw new RuntimeException("exception");
                }
                s.setTestStr("test_str5");
                mapper.updateById(s);
            });
        } catch (Throwable e) {
            System.out.println("更新失败!");
            flag.set(true);
        } finally {
            // 切换回主线程执行
            countDownLatch.countDown();
        }
        try {
            // 等待主线程执行
            mainLatch.await();
        } catch (Throwable e) {
            System.out.println("主线程超时!");
            flag.set(true);
        }

        if (flag.get()) {
            long end = System.currentTimeMillis();
            System.out.println("子线程回滚:" + Thread.currentThread().getName() + "耗时:" + (end - start));
            dataSourceTransactionManager.rollback(transactionStatus);
        } else {
            long end = System.currentTimeMillis();
            System.out.println("子线程提交:" + Thread.currentThread().getName() + "耗时:" + (end - start));
            dataSourceTransactionManager.commit(transactionStatus);
        }

    }