如何多线程下载一个文件

设计

1. 把一个文件下载流分成 n 份，即 n 个线程各下载一份
2. 等待所有线程搞定，之后将 n 个文件合并为一个

实例

以下载 QQ2020 PC为例 ，大小82MB

1. 留意HTTP协议，先查看下它的HTTP response

文件类型是octet-stream ，支持范围请求 Accept-Ranges，大小 86025424 字节。

2. HTTP request 构建，关键在于 headers 的 Range 参数，比如 Range: bytes=0-1023 表示只获取文件从 0 到 1023（包括）共1024字节的部分文件

3. 实现 1 ，先发送一个 head 请求，获取文件的长度 contentLength

        HttpRequest headReq = HttpRequest
                        .newBuilder(qqUri)
                        .method("HEAD", BodyPublishers.noBody() )
                        .build();
        HttpResponse<Void> res = client.send(headReq, BodyHandlers.discarding());
        long contentLength = res.headers().firstValueAsLong("content-length").orElseThrow();

4. 实现 2，构建多个请求，这里是 5 个

        long partLength = contentLength / 5 + 1;
        HttpRequest.Builder baseRequest = HttpRequest.newBuilder(qqUri).GET();

        CountDownLatch latch = new CountDownLatch(5);
        List<Callable<HttpResponse<Path>>> tasks = new ArrayList<>();
        int start = 0;
        for (int i = 0; i < 5; i++) {
            String range = "bytes=" + start + "-" + (start += partLength);
            var req = baseRequest.copy().header("Range", range).build();
            String tmpPath = localpath + i;
            tasks.add(() -> {
                var r =  client.send(req, BodyHandlers.ofFile(Path.of(tmpPath)));
                latch.countDown();
                return r;
            });
            start += 1;
        }

5. 启动多个并发请求

        var futures = executor.invokeAll(tasks);

6. 合并文件

        Path target = Path.of(localpath);
        for (int i = 0; i < 5; i++) {
            var source = Path.of(localpath + i);
            Files.write(target, Files.readAllBytes(source), StandardOpenOption.APPEND, StandardOpenOption.CREATE);
            source.toFile().delete();
        }

测试

对比了 5 线程下载和单线程下载，家里网络波动较大，这里看看就好（网速丢人了🤦‍）；合并后的安装包可以正常安装😘

完整代码

import java.net.URI;
import java.net.http.HttpClient;
import java.net.http.HttpRequest;
import java.net.http.HttpRequest.BodyPublishers;
import java.net.http.HttpResponse;
import java.net.http.HttpResponse.BodyHandlers;
import java.nio.file.Files;
import java.nio.file.Path;
import java.nio.file.StandardOpenOption;
import java.time.Duration;
import java.util.ArrayList;
import java.util.List;
import java.util.concurrent.Callable;
import java.util.concurrent.CountDownLatch;
import java.util.concurrent.ExecutorService;
import java.util.concurrent.Executors;

public class DownloadFile {

    static final String url = "https://945e4466dae548ca4396daa2bec9b35c.dlied1.cdntips.net/dlied1.qq.com/qqweb/PCQQ/PCQQ_EXE/PCQQ2020.exe?mkey=5f45034a7b0500bc&f=255a&cip=123.5.38.73&proto=https&access_type=$header_ApolloNet";
    static final URI qquri = URI.create(url);
    static final String localpath = "f:\\data\\qq.exe";
    static final String localpath2 = "f:\\data\\qq-2.exe";
    static ExecutorService executor = Executors.newFixedThreadPool(7);

    static void downloadFileBy5Threads() throws Exception {

        HttpClient client = HttpClient.newBuilder().connectTimeout(Duration.ofSeconds(5)).build();

        long startTime = System.currentTimeMillis();

        // 1. 得到文件长度
        HttpRequest headReq = HttpRequest.newBuilder(qquri).method("HEAD", BodyPublishers.noBody()).build();
        HttpResponse<Void> res = client.send(headReq, BodyHandlers.discarding());
        long contentLength = res.headers().firstValueAsLong("content-length").orElseThrow();
        
        System.out.println("文件内容长度: " + contentLength);

        // 2. 把文件划分为5分, 构建 5 个请求
        long partLength = contentLength / 5 + 1;
        HttpRequest.Builder baseRequest = HttpRequest.newBuilder(qquri).GET();

        CountDownLatch latch = new CountDownLatch(5);
        List<Callable<HttpResponse<Path>>> tasks = new ArrayList<>();
        int start = 0;
        for (int i = 0; i < 5; i++) {
            String range = "bytes=" + start + "-" + (start += partLength);
            var req = baseRequest.copy().header("Range", range).build();
            String tmpPath = localpath + i;
            tasks.add(() -> {
                System.out.println("线程已启动 " + range);
                var r = client.send(req, BodyHandlers.ofFile(Path.of(tmpPath)));
                latch.countDown();
                return r;
            });
            start += 1;
        }
        
        // 3. 执行并等待所有线程完成
        var futures = executor.invokeAll(tasks);
        latch.await();

        // 4. 把五个文件合并为一个
        Path target = Path.of(localpath);
        for (int i = 0; i < 5; i++) {
            var source = Path.of(localpath + i);
            Files.write(target, Files.readAllBytes(source), StandardOpenOption.APPEND, StandardOpenOption.CREATE);
            source.toFile().delete();
        }

        System.out.println("文件多线程下载完成，用时 " + (System.currentTimeMillis() - startTime) + "ms");

    }

    static void downloadFileByOneThread() throws Exception {


        HttpClient client = HttpClient.newBuilder().connectTimeout(Duration.ofSeconds(5)).build();

        long startTime = System.currentTimeMillis();

        var req = HttpRequest.newBuilder(qquri).GET().build();
        client.send(req, BodyHandlers.ofFile(Path.of(localpath2)));

        System.out.println("文件单线程下载完成。用时 " + (System.currentTimeMillis() - startTime) + "ms");
    }

    public static void main(String[] args) throws Exception {

        downloadFileBy5Threads();
        downloadFileByOneThread();

    }
}