大文件断点上传

demo

大文件上传

1. 使用Blob.prototype.slice方法将大文件分割成指定大小文件切片。
2. 使用formData并行上传所有文件切片数据（由于并行上传，将文件切片的顺序告诉服务端）。
3. 所有切片全部上传后通知服务端进行文件合并。
4. 服务端使用multiparty处理前端传的formData数据，存储所有文件切片数据。
5. 服务端根据文件切片的顺序使用读写流进行文件合并，合并成功后删除文件切片数据。

前端代码

import React, { useState } from 'react';
import { Button, message } from 'antd';
import request from '@/utils/request';

const SIZE = 10 * 1024 * 1024; // 单个文件切片大小
const BigFileUpload = () => {
  const [uploadStatus, setUploadStatus] = useState(''); // 上传的状态
  const [selectFile, setSelectFile] = useState<any>(null); // 选择的文件

  // 选择上传文件时存储文件信息
  const onUploadChange = (e: any) => {
    const [file] = e.target.files;
    if (file) {
      setSelectFile(file);
    }
  };

  // 生成文件切片
  const createFileChunk = (file: any) => {
    const fileChunkList = [];
    let cur = 0;
    while (cur < file.size) {
      fileChunkList.push({ file: file.slice(cur, cur + SIZE) });
      cur += SIZE;
    }
    return fileChunkList;
  };

  // 上传文件
  const uploadFile = () => {
    if (selectFile) {
      setUploadStatus('loading');
      const fileChunkList = createFileChunk(selectFile);
      const fileChunkData: any = fileChunkList.map(({ file }, index) => ({
        index,
        chunk: file,
        size: file.size
      }));
      uploadChunks(fileChunkData);
    }
  };

  // 上传切片
  const uploadChunks = async (fileChunkData: any) => {
    const requestList = fileChunkData
      .map(({ chunk, index }: { chunk: any; index: number }) => {
        const formData = new FormData();
        formData.append('chunk', chunk);
        formData.append('filename', selectFile.name);
        formData.append('index', String(index));
        return { formData, index };
      })
      .map(({ formData }: { formData: any; index: number }) =>
        request({
          url: '/upload',
          method: 'POST',
          data: formData
        })
      );
    await Promise.all(requestList);
    await mergeFile();
  };

  // 通知服务端合并切片
  const mergeFile = async () => {
    await request({
      url: '/merge',
      method: 'POST',
      data: JSON.stringify({
        size: SIZE,
        filename: selectFile.name
      })
    });
    message.success('上传成功');
    setUploadStatus('');
  };

  return (
    <div>
      {/* 正在上传中时disabled */}
      <input
        type="file"
        onChange={onUploadChange}
        disabled={uploadStatus === 'loading'}
      />
      {/* 没有选择文件或正在上传中时disabled */}
      <Button
        onClick={uploadFile}
        disabled={!selectFile || uploadStatus === 'loading'}
      >
        上传
      </Button>
    </div>
  );
};

export default BigFileUpload;

服务端代码

const multiparty = require("multiparty");
const fse = require("fs-extra");
const path = require("path");

// 大文件存储目录
const UPLOAD_DIR = path.resolve(__dirname, "..", "files");

// 写入文件流
const pipeStream = (path, writeStream) =>
  new Promise(resolve => {
    const readStream = fse.createReadStream(path);
    readStream.on("end", () => {
      fse.unlinkSync(path);
      resolve();
    });
    readStream.pipe(writeStream);
  });

// 提取POST请求参数
const resolvePost = req =>
  new Promise(resolve => {
    let chunk = "";
    req.on("data", data => {
      chunk += data;
    });
    req.on("end", () => {
      resolve(JSON.parse(chunk));
    });
  });

// 创建临时文件夹用于临时存储chunk (添加 chunkDir 前缀与文件名做区分)
const getChunkDir = fileName => path.resolve(UPLOAD_DIR, `chunkDir_${fileName}`);

// 合并切片
const mergeFileChunk = async (filePath, filename, size) => {
  const chunkDir = getChunkDir(filename);
  const chunkPaths = await fse.readdir(chunkDir);
  // 根据切片下标进行排序，否则直接读取目录的获得的顺序会错乱
  chunkPaths.sort((a, b) => a - b);

  // 并发写入文件
  await Promise.all(
    chunkPaths.map((chunkPath, index) =>
      pipeStream(
        path.resolve(chunkDir, chunkPath),
        // 根据 size 在指定位置创建可写流
        fse.createWriteStream(filePath, {
          start: index * size
        })
      )
    )
  );
  // 合并后删除保存切片的目录
  fse.rmdirSync(chunkDir);
};

module.exports = class {
  // 处理文件切片
  async handleFormData(req, res) {
    const multipart = new multiparty.Form();
    multipart.parse(req, async (err, fields, files) => {
      if (err) {
        console.error(err);
        res.status = 500;
        res.end(
          JSON.stringify({
            code: 100,
            message: "file error"
          })
        );
        return;
      }
      const [chunk] = files.chunk;
      const [filename] = fields.filename;
      const [index] = fields.index;
      const filePath = path.resolve(
        UPLOAD_DIR,
        `${filename}`
      ); // 最终合并后的文件路径
      const chunkDir = getChunkDir(filename); // 存放chunk的文件夹路径
      const chunkPath = path.resolve(chunkDir, index); // 存放每个切片文件的路径

      // 最终合并后的文件已经存在直接返回
      if (fse.existsSync(filePath)) {
        res.end(
          JSON.stringify({
            code: 0,
            message: "file exist"
          })
        );
        return;
      }

      // 切片存在直接返回
      if (fse.existsSync(chunkPath)) {
        res.end(
          JSON.stringify({
            code: 0,
            message: "chunk exist"
          })
        );
        return;
      }

      // 切片目录不存在，创建切片目录
      if (!fse.existsSync(chunkDir)) {
        await fse.mkdirs(chunkDir);
      }

      await fse.move(chunk.path, chunkPath);
      res.end(
        JSON.stringify({
          code: 0,
          message: "success"
        })
      );
    });
  }

  // 合并切片
  async handleMerge(req, res) {
    const data = await resolvePost(req);
    const { filename, size } = data;
    const filePath = path.resolve(UPLOAD_DIR, `${filename}`);
    await mergeFileChunk(filePath, filename, size);
    res.end(
      JSON.stringify({
        code: 0,
        message: "success"
      })
    );
  }
};

显示上传进度条

• 使用axios的onUploadProgress方法获取到切片上传的进度，可以独立显示每个文件切片的上传进度。
• 根据每个切片的上传进度计算出整个文件的上传进度可以显示整个文件的上传进度。

断点续传

• 记住已上传的切片，下次上传时跳过已上传的文件切片。
  1. 前端使用localStorage存储已上传的切片（切换浏览器后失效）
  2. 服务端存储，前端每次上传前询问服务端已上传的部分（建议使用）。
• 使用spark-md5计算文件hash值作为文件的唯一标识，因为使用文件名作为标识时，文件名一修改就失去效果。计算hash是非常耗费时间，使用web-worker在worker线程计算hash，这样用户仍可以在主界面正常的交互。
• 暂停上传使用axios的CancelToken取消请求。
• 恢复上传时询问服务端已经成功上传的文件切片，重新上传时过滤掉已经上传的文件切片。

前端代码

import React, { useEffect, useState, useRef } from 'react';
import { Button, message, Table, Progress } from 'antd';
import axios from 'axios';
import request from '@/utils/request';

const { CancelToken } = axios;
const SIZE = 100 * 1024 * 1024; // 单个文件切片大小
const BigFileContinueUpload = () => {
  const cancelRequestRef = useRef<any>([]); // 取消请求的存储
  const [uploadStatus, setUploadStatus] = useState(''); // 上传的状态
  const [selectFile, setSelectFile] = useState<any>(null); // 选择的文件
  const [selectFileHash, setSelectFileHash] = useState<any>(null); // 选择的文件hash值
  const [chunkData, setChunkData] = useState<any>([]); // 切片文件数据
  const [totalPercentage, setTotalPercentage] = useState<number>(0); // 上传总进度

  // 监听文件切片上传进度影响总进度
  useEffect(() => {
    let percentage = 0;
    chunkData.forEach((item: any) => (percentage += item.percentage));
    setTotalPercentage(percentage / 3);
  }, [chunkData]);

  // 选择上传文件时存储文件信息
  const onUploadChange = (e: any) => {
    const [file] = e.target.files;
    if (file) {
      setSelectFile(file);
    }
  };

  // 生成文件hash
  const calculateHash = (fileChunkList: any) => {
    return new Promise(resolve => {
      const worker = new Worker('/hash.js');
      worker.postMessage({ fileChunkList });
      worker.onmessage = e => {
        const { hash } = e.data;
        if (hash) {
          resolve(hash);
        }
      };
    });
  };

  // 生成文件切片
  const createFileChunk = (file: any) => {
    const fileChunkList = [];
    let cur = 0;
    while (cur < file.size) {
      fileChunkList.push({ file: file.slice(cur, cur + SIZE) });
      cur += SIZE;
    }
    return fileChunkList;
  };

  // 上传文件
  const uploadFile = async () => {
    if (selectFile) {
      setUploadStatus('loading');
      const fileChunkList = createFileChunk(selectFile);
      const fileHash = await calculateHash(fileChunkList);
      setSelectFileHash(fileHash);
      const fileChunkData: any = fileChunkList.map(({ file }, index) => ({
        index,
        chunk: file,
        size: file.size,
        percentage: 0
      }));
      setChunkData(fileChunkData);
      uploadChunks(fileChunkData, fileHash);
    }
  };

  // 上传切片
  const uploadChunks = async (
    fileChunkData: any,
    fileHash: any,
    hasUploadedChunk: any = []
  ) => {
    const requestList = fileChunkData
      // 断点续传时过滤掉已经上传的文件切片
      .filter(
        ({ index }: { index: number }) =>
          !hasUploadedChunk.includes(String(index))
      )
      .map(({ chunk, index }: { chunk: any; index: number }) => {
        const formData = new FormData();
        formData.append('chunk', chunk);
        formData.append('filename', selectFile.name);
        formData.append('filehash', fileHash);
        formData.append('index', String(index));
        return { formData, index };
      })
      .map(({ formData, index }: { formData: any; index: number }) =>
        request({
          url: '/upload',
          method: 'POST',
          data: formData,
          // 处理上传进度
          onUploadProgress: function (progressEvent: any) {
            const percentCompleted = Math.round(
              (progressEvent.loaded * 100) / progressEvent.total
            );
            setChunkData((prev: any) => {
              const newData = [...prev];
              newData[index].percentage = percentCompleted;
              return newData;
            });
          },
          cancelToken: new CancelToken((cancel: any) => {
            cancelRequestRef.current.push(cancel);
          })
        })
      );
    await Promise.all(requestList);
    await mergeFile(fileHash);
  };

  // 通知服务端合并切片
  const mergeFile = async (fileHash: any) => {
    await request({
      url: '/merge',
      method: 'POST',
      data: JSON.stringify({
        size: SIZE,
        filename: selectFile.name,
        fileHash: fileHash
      })
    });
    message.success('上传成功');
    setUploadStatus('');
  };

  // 暂停上传
  const pauseUpload = () => {
    (cancelRequestRef.current || []).forEach((cancel: any) => cancel());
  };

  // 继续上传
  const resumeUpload = async () => {
    const res = await request({
      url: '/verify',
      method: 'POST',
      data: JSON.stringify({
        fileHash: selectFileHash
      })
    });
    uploadChunks(chunkData, selectFileHash, res.data?.uploadedList || []);
  };

  const columns = [
    {
      title: '切片',
      dataIndex: 'index',
      render: (value: number) => `${selectFile.name}-${value}`
    },
    {
      title: '切片大小',
      dataIndex: 'size'
    },
    {
      title: '进度',
      dataIndex: 'percentage',
      render: (value: number) => <Progress percent={value} size="small" />
    }
  ];

  return (
    <div>
      {/* 正在上传中时disabled */}
      <input
        type="file"
        onChange={onUploadChange}
        disabled={uploadStatus === 'loading'}
      />
      {/* 没有选择文件或正在上传中时disabled */}
      <Button
        onClick={uploadFile}
        disabled={!selectFile || uploadStatus === 'loading'}
        type="primary"
      >
        上传
      </Button>
      <Button onClick={pauseUpload}>暂停上传</Button>
      <Button onClick={resumeUpload}>继续上传</Button>
      <h6>总上传进度</h6>
      <Progress percent={totalPercentage} style={{ width: '90%' }} />
      <h6>文件切片上传进度</h6>
      <Table columns={columns} dataSource={chunkData} />
    </div>
  );
};

export default BigFileContinueUpload;

服务端代码

/**
 * 大文件断点续传
 */
/* eslint-disable */
const multiparty = require("multiparty");
const fse = require("fs-extra");
const path = require("path");

// 大文件存储目录
const UPLOAD_DIR = path.resolve(__dirname, "..", "files");

// 写入文件流
const pipeStream = (path, writeStream) =>
  new Promise(resolve => {
    const readStream = fse.createReadStream(path);
    readStream.on("end", () => {
      fse.unlinkSync(path);
      resolve();
    });
    readStream.pipe(writeStream);
  });

// 提取POST请求参数
const resolvePost = req =>
  new Promise(resolve => {
    let chunk = "";
    req.on("data", data => {
      chunk += data;
    });
    req.on("end", () => {
      resolve(JSON.parse(chunk));
    });
  });

// 创建临时文件夹用于临时存储chunk (添加 chunkDir 前缀与文件hash做区分)
const getChunkDir = fileHash => path.resolve(UPLOAD_DIR, `chunkDir_${fileHash}`);

// 返回已上传的所有切片名
const createUploadedList = async fileHash =>
  fse.existsSync(getChunkDir(fileHash))
    ? await fse.readdir(getChunkDir(fileHash))
    : [];

// 合并切片
const mergeFileChunk = async (filePath, size, fileHash) => {
  const chunkDir = getChunkDir(fileHash);
  const chunkPaths = await fse.readdir(chunkDir);
  // 根据切片下标进行排序，否则直接读取目录的获得的顺序会错乱
  chunkPaths.sort((a, b) => a - b);

  // 并发写入文件
  await Promise.all(
    chunkPaths.map((chunkPath, index) =>
      pipeStream(
        path.resolve(chunkDir, chunkPath),
        // 根据 size 在指定位置创建可写流
        fse.createWriteStream(filePath, {
          start: index * size
        })
      )
    )
  );
  // 合并后删除保存切片的目录
  fse.rmdirSync(chunkDir);
};

module.exports = class {
  // 获取已上传的文件切片
  async handleVerifyUpload(req, res) {
    const data = await resolvePost(req);
    const { fileHash } = data;
    res.end(
      JSON.stringify({
        uploadedList: await createUploadedList(fileHash)
      })
    );
  }

  // 处理文件切片
  async handleFormData(req, res) {
    const multipart = new multiparty.Form();
    multipart.parse(req, async (err, fields, files) => {
      if (err) {
        console.error(err);
        res.status = 500;
        res.end(
          JSON.stringify({
            code: 100,
            message: "file error"
          })
        );
        return;
      }
      const [chunk] = files.chunk;
      const [filename] = fields.filename;
      const [index] = fields.index;
      const [filehash] = fields.filehash;
      const filePath = path.resolve(
        UPLOAD_DIR,
        `${filename}`
      ); // 最终合并后的文件路径
      const chunkDir = getChunkDir(filehash); // 存放chunk的文件夹路径
      const chunkPath = path.resolve(chunkDir, index); // 存放每个切片文件的路径

      // 最终合并后的文件已经存在直接返回
      if (fse.existsSync(filePath)) {
        res.end(
          JSON.stringify({
            code: 0,
            message: "file exist"
          })
        );
        return;
      }

      // 切片存在直接返回
      if (fse.existsSync(chunkPath)) {
        res.end(
          JSON.stringify({
            code: 0,
            message: "chunk exist"
          })
        );
        return;
      }

      // 切片目录不存在，创建切片目录
      if (!fse.existsSync(chunkDir)) {
        await fse.mkdirs(chunkDir);
      }

      await fse.move(chunk.path, chunkPath);
      res.end(
        JSON.stringify({
          code: 0,
          message: "success"
        })
      );
    });
  }

  // 合并切片
  async handleMerge(req, res) {
    const data = await resolvePost(req);
    const { filename, size, fileHash } = data;
    const filePath = path.resolve(UPLOAD_DIR, `${filename}`);
    await mergeFileChunk(filePath, size, fileHash);
    res.end(
      JSON.stringify({
        code: 0,
        message: "success"
      })
    );
  }
};