前后端实现分片上传大文件

一、node后端

1.上传接口存储位置，我是合并接口，前端判断大于5M就分片上传，创建临时文件，后续合并之后就删除临时文件夹，存放的路径是项目的src/uploads

// 设置multer存储配置
const storage = multer.diskStorage({
  destination: function (req, file, cb) {
    // 设置文件上传的目录为项目根目录下的 src/uploads
    const uploadDir = path.join(__dirname, '../uploads');
    // 确保主上传目录存在
    if (!fs.existsSync(uploadDir)) {
      fs.mkdirSync(uploadDir, { recursive: true });
    }

    // 如果有 fileId，创建一个以 fileId 为名的临时文件夹
    const fileId = req.body.fileId;
    if (fileId) {
      const tempDir = path.join(uploadDir, fileId);
      if (!fs.existsSync(tempDir)) {
        fs.mkdirSync(tempDir, { recursive: true });
      }
      cb(null, tempDir);
    } else {
      cb(null, uploadDir);
    }
  },
  filename: function (req, file, cb) {
    const originalName = req.body.originalName || file.originalname;
    // 检查是否有分片信息
    if (req.body.index !== undefined) {
      // 使用原始文件名和分片索引构建文件名
      const chunkIndex = req.body.index;
      cb(null, `${originalName}-chunk-${chunkIndex}`); // 构建分片文件名
    } else {
      // 没有分片信息，使用原始文件名
      cb(null, originalName); // 使用原始文件名
    }
  }
});
// 创建multer实例并指定存储配置
const upload = multer({ storage }).fields([
  { name: 'file', maxCount: 1 }, // 用于普通上传
  { name: 'chunk', maxCount: 1 } // 用于分片上传
]);

2.上传接口，获取分片信息并存入到数据库里面

// 上传文件
const uploadFile = async (req, res, next) => {
  try {
    const { files } = req
    let file = files['file'] ? files['file'][0] : null; // 普通上传的文件
    let chunk = files['chunk'] ? files['chunk'][0] : null; // 分片上传的文件

    // 确定是普通文件还是分片
    const fileToProcess = chunk || file;
    // 假设使用了multer中间件处理文件上传，并且文件信息在req.file中
    if (!fileToProcess) {
      return next();
    }
    const { index, totalChunks, fileId, originalName, userId, folderId } = req.body;
    const chunkFile = fileToProcess.path;

    // 如果只有一个分片或没有分片信息，则直接保存文件信息
    if (!totalChunks || totalChunks === '1') {
      const fileId = await saveFileInfo(userId, folderId, originalName, chunkFile, fileToProcess.size);
      res.success({ data: '文件上传成功', fileId });
    } else {
      // 处理分片上传
      const insertQuery = 'INSERT INTO file_chunks (fileId, chunkIndex, totalChunks, filePath, originalFilename) VALUES (?, ?, ?, ?, ?)';
      await db.query(insertQuery, [fileId, index, totalChunks, chunkFile, originalName]);
      res.success({ data: '分片上传成功' });
    }
  } catch (error) {
    next('文件上传失败');
  }
};

3.保存分片到数据库

// 保存文件到数据库
const saveFileInfo = async (userId, folderId, originalName, filePath, fileSize) => {
  const file_name = originalName.slice(0, originalName.lastIndexOf('.'));
  const file_type = originalName.slice(originalName.lastIndexOf('.') + 1);
  const [fileType, typeId] = getTypeIdByExtension(file_type);
  const url = `/uploads/${path.basename(filePath)}`;

  const sql = `INSERT INTO files (userId, folderId, name, size, path, type, typeId, fileType, url) VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?)`;
  const [result] = await db.query(sql, [userId, folderId, file_name, fileSize, filePath, file_type, typeId, fileType, url]);
  return result.insertId;
}

4、合并分片并删除临时文件夹

// 合并分片
const mergeFile = async (req, res, next) => {
  const { fileId, totalChunks, originalName, userId, folderId } = req.body;
  const targetDir = path.join(__dirname, '../uploads', fileId);
  const filePath = path.join(__dirname, '../uploads', originalName);

  // 检查是否所有分片都已上传
  const [rows] = await db.query('SELECT COUNT(*) AS count FROM file_chunks WHERE fileId = ?', [fileId]);
  if (rows[0].count == totalChunks) {
    // 合并文件
    fs.writeFileSync(filePath, '');
    for (let i = 0; i < totalChunks; i++) {
      const currentChunkPath = path.join(targetDir, `${originalName}-chunk-${i}`);
      fs.appendFileSync(filePath, fs.readFileSync(currentChunkPath));
      fs.unlinkSync(currentChunkPath); // 删除分片
    }
    fs.rmdirSync(targetDir); // 删除目录

    const fileSize = fs.statSync(filePath).size; // 获取合并后文件的大小
    const fileId = await saveFileInfo(userId, folderId, originalName, filePath, fileSize);
    res.success({ data: '文件合并成功', fileId });
  } else {
    res.status(400).json({ message: '所有的分片未全部上传.' });
  }
};

二、前端react+ts实现分片

1.上传方法

const handleUpload = async (file: RcFile) => {
    const chunkSize = 5 * 1024 * 1024 // 5MB chunk size
    const fileSize = file.size
    console.log('🚀 ~ handleUpload ~ fileSize:', fileSize, chunkSize)
    const fileId = Date.now().toString() // Generate a unique ID for the file

    setProgress(0) // Reset progress
    setUploading(true) // Start uploading

    if (fileSize > chunkSize) {
      // If file size is greater than 50MB, do chunk upload
      const totalChunks = Math.ceil(fileSize / chunkSize)
      for (let i = 0; i < totalChunks; i++) {
        const chunk = file.slice(i * chunkSize, (i + 1) * chunkSize)
        const chunkInfo: FileUpload = {
          userId,
          folderId,
          file,
          chunk,
          index: i,
          totalChunks,
          fileId
        }
        console.log('🚀 ~ handleUpload ~ chunkInfo:', chunkInfo)
        await uploadsFile(chunkInfo)
        setProgress((prevProgress) => prevProgress + 100 / totalChunks)
      }

      // All chunks uploaded, now notify the server to merge them
      try {
        await mergeChunks({
          fileId,
          totalChunks,
          userId,
          folderId,
          originalName: file.name
        })
        message.success('文件合并成功！')
      } catch (error) {
        message.error('文件合并失败！')
        console.error('Error merging file chunks:', error)
      }
      // ... (You might need to implement a separate API endpoint for merging chunks)
    } else {
      // If file size is less than or equal to 50MB, do normal upload
      const fileInfo: FileUpload = {
        userId,
        folderId,
        file
      }
      await uploadsFile(fileInfo)
      setProgress(100)
    }

    setUploading(false) // Finish uploading
    message.success('文件上传成功！')
  }

2.接口调用，在body中添加chunk等分片信息来区分是不是大文件分片上传

export const uploadsFile = (fileInfo: FileUpload) => {
  const formData = new FormData()
  formData.append('userId', fileInfo.userId.toString())
  formData.append('folderId', fileInfo.folderId.toString())
  formData.append('originalName', fileInfo.file.name) // 原始文件名
  // 如果存在分片信息，则为分片上传，否则为普通上传
  if (
    fileInfo.chunk &&
    fileInfo.index !== undefined &&
    fileInfo.totalChunks &&
    fileInfo.fileId
  ) {
    // 分片上传
    formData.append('index', fileInfo.index.toString())
    formData.append('totalChunks', fileInfo.totalChunks.toString())
    formData.append('fileId', fileInfo.fileId)
    formData.append('chunk', fileInfo.chunk)
  } else {
    // 普通上传
    formData.append('file', fileInfo.file)
  }

  console.log('🚀 ~ uploadsFile ~ formData:', formData)
  return request.post(`/files/upload`, formData)
}
// 合并分片
export const mergeChunks = (chunkInfo: ChunkInfo) => {
  return request.post<string>(`/files/merge`, chunkInfo, { showMessage: true })
}