大文件切片上传（附前后端代码）

本文分为服务端和前端两个部分来实现大文件上传的逻辑

服务端

技术选型和环境准备

• nodejs v16.17.1
• 服务端框架：nest ^10.0.0
• 包管理工具：yarn 1.22.18
• multiparty ^4.2.3：处理上传接口的文件参数

为了方便调试，需要跨域配置

app.enableCors({
  origin: '*', // 允许所有来源的请求
  methods: 'GET,HEAD,PUT,PATCH,POST,DELETE',
  allowedHeaders: 'Content-Type,Authorization',
});

const fs = require('fs');
const multiparty = require('multiparty');

export const host = '127.0.0.1';
export const port = 3000;
export const HOSTNAME = `${host}:${port}`;
export const SERVER_PATH = `${__dirname}/upload`;
// 检测文件是否已经存在
function exists(path) {
  return new Promise((resolve, reject) => {
    fs.access(path, fs.constants.F_OK, (err) => {
      if (err) {
        resolve(false);
        return;
      }
      resolve(true);
    });
  });
}

// 将传进来的文件数据写入服务器
// form-data格式的数据将以流的形式写入
// BASE64格式数据则直接将内容写入
function writeFile(serverPath, file, isStream) {
  return new Promise((resolve, reject) => {
    if (isStream) {
      try {
        const readStream = fs.createReadStream(file.path);
        const writeStream = fs.createWriteStream(serverPath);
        readStream.pipe(writeStream);
        readStream.on('end', () => {
          resolve({
            result: true,
            message: '上传成功！',
          });
          fs.unlinkSync(file.path);
        });
      } catch (err) {
        resolve({
          result: false,
          message: err,
        });
      }
    } else {
      fs.writeFile(serverPath, file, (err) => {
        if (err) {
          resolve({
            result: false,
            message: err,
          });
          return;
        }
        resolve({
          result: true,
          message: '上传成功！',
        });
      });
    }
  });
}

//定义延迟函数
function delay(interval) {
  typeof interval !== 'number' ? (interval = 1000) : null;
  return new Promise((resolve, reject) => {
    setTimeout(function () {
      resolve({});
    }, interval);
  });
}

function mergeFiles(hash, count, name) {
  return new Promise(async (resolve, reject) => {
    const dirPath = `${SERVER_PATH}/${hash}`;
    if (!fs.existsSync(dirPath)) {
      reject('还没上传文件，请先上传文件');
      return;
    }
    const fileList = fs.readdirSync(dirPath);
    if (fileList.length < count) {
      reject('文件还未上传完成，请稍后再试');
      return;
    }
    let suffix;
    fileList
      .sort((a, b) => {
        const reg = /_(\d+)/;
        return Number(reg.exec(a)[1]) - Number(reg.exec(b)[1]);
      })
      .forEach((item) => {
        // !suffix ? (suffix = /.([0-9a-zA-Z]+)$/.exec(item)[1]) : null;
        //将每个文件读取出来并append到以hash命名的新文件中
        fs.appendFileSync(
          `${SERVER_PATH}/${name}`,
          fs.readFileSync(`${dirPath}/${item}`),
        );
        fs.unlinkSync(`${dirPath}/${item}`); //删除切片文件
      });

    await delay(1000); //等待1秒后删除新产生的文件夹
    fs.rmdirSync(dirPath);
    resolve({
      path: `${HOSTNAME}/upload/${name}.${suffix}`,
      filename: `${name}.${suffix}`,
    });
  });
}

// 利用multiparty插件解析前端传来的form-data格式的数据，并上传至服务器
function multipartyUpload(req, autoUpload) {
  const config: any = {
    maxFieldsSize: 200 * 1024 * 1024,
  };
  if (autoUpload) config.uploadDir = SERVER_PATH;
  return new Promise((resolve, reject) => {
    new multiparty.Form(config).parse(req, (err, fields, files) => {
      if (err) {
        reject(err);
        return;
      }
      resolve({
        fields,
        files,
      });
    });
  });
}

export { exists, writeFile, mergeFiles, multipartyUpload };

import { Controller, Get, Post, Req } from '@nestjs/common';
import { Request } from 'express';
import { AppService } from './app.service';
import {
  HOSTNAME,
  SERVER_PATH,
  exists,
  mergeFiles,
  multipartyUpload,
  writeFile,
} from './utils';

const fs = require('fs');

@Controller('')
export class AppController {
  constructor(private readonly appService: AppService) {}

  @Post('upload_chunk')
  async upload(@Req() request: Request): Promise<any> {
    try {
      const { files, fields }: any = await multipartyUpload(request, false);
      // console.log(files, fields);
      const file = (files && files.file[0]) || {};
      const filename = (fields && fields.filename[0]) || '';
      const [, hash] = /^([^_]+)_(\d+)/.exec(filename);
      const dirPath = `${SERVER_PATH}/${hash}`;
      if (!fs.existsSync(dirPath)) {
        fs.mkdirSync(dirPath);
      }
      const filePath = `${dirPath}/${filename}`;
      const isExists = await exists(filePath);
      if (isExists) {
        return {
          code: 0,
          message: '文件已经存在',
          originalFilename: filename,
          serverPath: filePath.replace(__dirname, HOSTNAME),
        };
      }
      await writeFile(filePath, file, true);
      return {
        code: 0,
        message: '文件上传成功',
        serverPath: filePath.replace(__dirname, HOSTNAME),
      };
    } catch (err) {
      console.error(err);
      return {
        code: 1,
        message: err.message,
      };
    }
  }

  @Post('upload_merge')
  async upload_merge(@Req() request: Request): Promise<any> {
    try {
      const { hash, count, name }: any = request.body;
      console.log(hash, count);
      const { path, filename }: any = await mergeFiles(hash, count, name);
      return {
        code: 0,
        message: '文件上传成功',
        path,
        filename,
      };
    } catch (err) {
      console.error(err);
      return {
        code: 1,
        message: err.message,
      };
    }
  }
}

前端

前端不使用框架，主要处理逻辑在文件分片，使用了第三库做逻辑处理

• axios v1.4.0：用于调用服务端接口发送服务端请求
• spark-md5 v3.0.2：根据文件内容生成hashCode

<!-- HTML -->
<input type="file" id="fileInput" />

// js
<script
src="https://cdnjs.cloudflare.com/ajax/libs/axios/1.4.0/axios.min.js"
integrity="sha512-uMtXmF28A2Ab/JJO2t/vYhlaa/3ahUOgj1Zf27M5rOo8/+fcTUVH0/E0ll68njmjrLqOBjXM3V9NiPFL5ywWPQ=="
crossorigin="anonymous"
referrerpolicy="no-referrer"
  ></script>
  <script
src="https://cdnjs.cloudflare.com/ajax/libs/spark-md5/3.0.2/spark-md5.min.js"
integrity="sha512-iWbxiCA4l1WTD0rRctt/BfDEmDC5PiVqFc6c1Rhj/GKjuj6tqrjrikTw3Sypm/eEgMa7jSOS9ydmDlOtxJKlSQ=="
crossorigin="anonymous"
referrerpolicy="no-referrer"
  ></script>

  <script>
  function retrieveHash(file) {
    return new Promise((resolve, reject) => {
      let spark = new SparkMD5.ArrayBuffer();
      let fr = new FileReader();
      fr.readAsArrayBuffer(file);
      fr.onload = (ev) => {
        spark.append(ev.target.result);
        let hash = spark.end();
        let suffix = /.([0-9a-zA-Z]+)$/.exec(file.name)[1];
        resolve({
          hash,
          suffix,
        });
      };
    });
  }

let complete = 0;
function uploadComplete(hash, count, name) {
  complete += 1;
  if (complete < count) return;
  console.log(hash, count);
  setTimeout(() => {
    axios
      .post('http://127.0.0.1:3000/upload_merge', {
        hash,
        count,
        name,
      })
      .then((res) => {
        console.log(res);
        // alert('上传成功了');
      })
      .catch((err) => {
        console.log(err);
      });
  }, 1000);
}

const fileInput = document.getElementById('fileInput');
fileInput.addEventListener('change', async (event) => {
  const files = event.target.files;

  for (let i = 0; i < files.length; i++) {
    const file = files[i];
    const maxSize = 1024 * 1024; // 1MB
    const count = Math.ceil(file.size / maxSize);
    const { hash, suffix } = await retrieveHash(file);

    const chunks = [];
    let index = 0;
    while (index < count) {
      chunks.push({
        file: file.slice(index * maxSize, (index + 1) * maxSize),
        filename: `${hash}_${index + 1}.${suffix}`,
      });
      index += 1;
    }
    chunks.forEach((item, index) => {
      let formData = new FormData();
      formData.append('file', item.file);
      formData.append('filename', item.filename);
      axios
        .post('http://127.0.0.1:3000/upload_chunk', formData, {
          contentType: 'multipart/form-data',
        })
        .then((res) => {
          uploadComplete(hash, count, file.name);
        })
        .catch((err) => {
          console.log(err);
        });
    });
  }
});

</script>

大文件上传时序图