前端文件上传和大文件切片上传

前言

github仓库

服务端

基于Express,有如下几个接口：

• upload: 普通上传文件
• upload_base64：普通图片base64
• upload_already：根据文件hash获取是否有已上传的部分文件列表
• upload_chunk: 分片上传
• upload_merge: 合并

const fs = require('fs-extra')
const path = require('node:path')
const kolorist = require('kolorist')
const express = require('express')
const bodyParser = require('body-parser')
const multiparty = require('multiparty')
const SparkMD5 = require('spark-md5')

const HOST = 'http://127.0.0.1'
const PORT = 8088
const HOSTNAME = `${HOST}:${PORT}`
const uploadDir = `${__dirname}/uploads`

const app = express()
app.listen(PORT, () => {
  console.log(`服务创建成功：${kolorist.blue(HOSTNAME)}`)
})
// 中间件
app.use((req, res, next) => {
  res.header('Access-Control-Allow-Origin', '*')
  req.method === 'OPTIONS' ? res.send('CURRENT SERVICES SUPPORT CROSS DOMAIN REQUESTS!') : next()
})

app.use(
  bodyParser.urlencoded({
    extended: false,
    limit: '1024mb'
  })
)

// 基于multiparty插件实现文件上传处理
const multiparty_upload = req => {
  return new Promise((resolve, reject) => {
    const form = new multiparty.Form({
      uploadDir, // 指定文件存储目录
      maxFieldsSize: 200 * 1024 * 1024
    })
    // 将请求参数传入，multiparty会进行相应处理
    form.parse(req, (err, fields, files) => {
      if (err) {
        reject(err)
        return
      }
      resolve({
        fields,
        files
      })
    })
  })
}

const writeChunkFile = (path, file) => {
  return new Promise((resolve, reject) => {
    try {
      const readStream = fs.createReadStream(file.path)
      const writeStream = fs.createWriteStream(path)
      readStream.pipe(writeStream)
      readStream.on('end', () => {
        resolve()
        fs.unlinkSync(file.path)
      })
    } catch (e) {
      reject(e)
    }
  })
}

// 上传文件
app.post('/upload', async (req, res) => {
  try {
    let { files } = await multiparty_upload(req)
    let file = (files.file && files.file[0]) || {}

    res.send(
      createSucess(
        {
          originalFilename: file.originalFilename,
          servicePath: file.path.replace(__dirname, HOSTNAME).replace(/\\/g, '/')
        },
        '上传成功'
      )
    )
  } catch (err) {
    res.send(createFailure(err))
  }
})

// 上传文件的base64
app.post('/upload_base64', async (req, res) => {
  try {
    let buffer = getBufferByBase64(req.body.file)

    const filename = req.body.filename
    // 把文件转md5
    const spark = new SparkMD5.ArrayBuffer()
    spark.append(buffer)
    const extName = getExtByFileName(filename)
    const path = `${uploadDir}/${spark.end()}.${extName}`
    // 写入文件
    await fs.writeFile(path, buffer)
    res.send(
      createSucess({
        originalFilename: filename,
        servicePath: path.replace(__dirname, HOSTNAME)
      })
    )
  } catch (err) {
    console.log('err', err)
    res.send(createFailure(err))
  }
})

// 根据文件hash获取是否有已上传的部分文件列表
app.get('/upload_already', async (req, res) => {
  const { HASH } = req.query
  const path = `${uploadDir}/${HASH}`
  let fileList = []
  try {
    fileList = await fs.readdir(path)
    fileList = fileList.sort((a, b) => {
      let reg = /_(\d+)/
      return reg.exec(a)[1] - reg.exec(b)[1]
    })
    res.send(createSucess({ fileList }))
  } catch (err) {
    res.send(createSucess({ fileList }))
  }
})
// 分片上传
app.post('/upload_chunk', async (req, res) => {
  try {
    let { files, fields } = await multiparty_upload(req)
    let file = (files.file && files.file[0]) || {}
    const filename = (fields.filename && fields.filename[0]) || ''
    if (!filename) {
      res.send(createFailure('文件名为空'))
      return
    }
    let [, hash] = /^([^_]+)_(\d+)/.exec(filename)
    let path = `${uploadDir}/${hash}`
    if (!fs.existsSync(path)) {
      fs.mkdir(path)
    }
    // 切片路径
    path = `${uploadDir}/${hash}/${filename}`
    const isExists = await fs.exists(path)
    if (isExists) {
      res.send(createFailure('切片文件已存在'))
      return
    }
    // 把切片存储到临时目录中
    await writeChunkFile(path, file)
    res.send(
      createSucess({
        originalFilename: filename,
        servicePath: path.replace(__dirname, HOSTNAME)
      })
    )
  } catch (err) {
    res.send(createFailure(err))
  }
})

// 把切片合并
const merge = async (hash, count) => {
  let path = `${uploadDir}/${hash}`
  let pathExists = await fs.exists(path)
  if (!pathExists) {
    throw 'hash对应的文件路径不存在'
  }
  let fileList = await fs.readdir(path)
  if (fileList.length < count) {
    throw '切片未上传完成'
  }
  fileList.sort((a, b) => {
    let reg = /_(\d+)/
    return reg.exec(a)[1] - reg.exec(b)[1]
  })
  // 文件后缀名
  let ext = ''
  for (let i = 0; i < fileList.length; i++) {
    const item = fileList[i]
    const itemPath = `${path}/${item}`
    ext = getExtByFileName(item)
    const buffer = await fs.readFile(itemPath)
    await fs.appendFile(`${uploadDir}/${hash}.${ext}`, buffer)
    await fs.unlink(itemPath)
  }
  await fs.rmdir(path)
  return {
    path: `${uploadDir}/${hash}.${ext}`,
    filename: `${hash}.${ext}`
  }
}

// api: 合并切片
app.post('/upload_merge', async (req, res) => {
  let { HASH, count } = req.body
  try {
    let { filename, path } = await merge(HASH, count)
    res.send(
      createSucess(
        {
          codeText: 'merge success',
          originalFilename: filename,
          servicePath: path.replace(__dirname, HOSTNAME)
        },
        '合并成功'
      )
    )
  } catch (err) {
    res.send(createFailure(err))
  }
})
// 静态文件夹
app.use('/uploads', express.static(path.join(__dirname, 'uploads')))
app.use((req, res) => {
  res.status(404)
  res.send('NOT FOUND!')
})

// 一些工具函数
// 定义接口成功时返回的数据结构
function createSucess(data, message = '成功') {
  return {
    code: 0,
    message,
    success: true,
    data
  }
}
// 定义接口失败时返回的数据结构
function createFailure(message, code = 1) {
  return {
    code,
    message,
    success: false
  }
}
// 获取后缀名
function getExtByFileName(fileName) {
  return /\.(\w+)$/.exec(fileName)[1]
}
// 把base64数据转为buffer
function getBufferByBase64(base64) {
  base64 = decodeURIComponent(base64)
  base64 = base64.replace(/^data:image\/\w+;base64,/, '')
  base64 = Buffer.from(base64, 'base64')
  return base64
}

前端

项目基于Vue3 + TS + Vite

<script setup lang="ts">
import { ref, shallowRef } from 'vue'
import SparkMD5 from 'spark-md5'
import uploadService from '@/api/services/uploadService'
const props = defineProps({
  // 是否使用分片上传
  useSlice: {
    type: Boolean,
    default: false
  },
  // 默认每个切片大小
  sliceSize: {
    type: Number,
    default: 1024 * 1024 * 1
  },
  // 并发池大小
  maxPoolsSize: {
    type: Number,
    default: 10
  }
})

const inputRef = shallowRef<HTMLInputElement>()
const isProcessing = ref(false)
const uploading = ref(false)
const uploadPencent = ref(0)

const handleClick = (e: MouseEvent) => {
  if (isProcessing.value) {
    alert('正在解析文件，请稍候')
    return
  }
  if (uploading.value) {
    alert('正在上传文件，请稍候')
    return
  }
  inputRef.value!.value = ''
  inputRef.value!.click()
}
const handleChange = async (e: Event) => {
  const files = (e.target as HTMLInputElement).files
  if (!files) return
  if (props.useSlice) {
    await sliceUpload(files[0])
    return
  }
  uploadFiles(Array.from(files))
}
const uploadFiles = async (files: File[]) => {
  if (files.length === 0) return
  try {
    uploading.value = true
    const file = files[0]
    // 限制上传文件大小
    // if (file.size > 2 * 1024 * 1024) {
    // }
    const res = await uploadService.upload(file, progress => {
      const { loaded, total } = progress
      uploadPencent.value = Math.round((loaded / total!) * 100)
    })
    console.log('res', res)
  } finally {
    uploading.value = false
  }
}
const onDrop = (e: DragEvent) => {
  e.preventDefault()
  let files = e.dataTransfer?.files
  if (!files || files.length === 0) return
  uploadFiles(Array.from(files))
}
const onDragover = (e: DragEvent) => {
  e.preventDefault()
}

// 分片上传
const sliceUpload = async (file: File) => {
  isProcessing.value = true
  const suffix: any = /\.([\w]+)$/.exec(file.name)![1]
  const fileHash = await getFileHash(file)
  let alreadyList: any[] = []
  const res = await uploadService.uploadAlready(fileHash)
  if (res.success) {
    alreadyList = res.data.fileList
  }
  let max = props.sliceSize // 每个切片的大小
  let count = Math.ceil(file.size / max) // 切片数量

  let chunks: any[] = [] // 所有切片
  let index = 0
  while (index < count) {
    chunks.push({
      file: file.slice(index * max, (index + 1) * max),
      filename: `${fileHash}_${index + 1}.${suffix}`
    })
    index++
  }
  index = 0
  isProcessing.value = false
  uploading.value = true
  let pools: any[] = [] // 并发池
  const maxPoolsSize = props.maxPoolsSize // 并发池最大值
  // 一个切片上传完成后执行的方法
  const chunkUploadComplete = async () => {
    index++
    uploadPencent.value = Math.round((index / count!) * 100)

    if (index < count) return
    // 全部切片上传完成
    uploadPencent.value = 100
    try {
      const res = await uploadService.uploadMerge(fileHash, count)
      if (res.success) {
        reset()
      }
    } catch (e) {}
  }
  let failChunks: any[] = [] // 上传失败的切片列表
  // 上传所有切片
  const uploadChunks = async (chunks: any[]) => {
    for (let idx = 0; idx < chunks.length; idx++) {
      let chunk = chunks[idx]
      // 已经上传过的
      if (alreadyList.length > 0 && alreadyList.includes(chunk.filename)) {
        chunkUploadComplete()
        continue
      }
      let task = uploadService.uploadChunk(chunk.file, chunk.filename)
      task
        .then(res => {
          if (res.success) {
            chunkUploadComplete()
          } else {
            failChunks.push(chunk)
          }
          // 执行完成,从池中移除
          const idx = pools.findIndex(x => x === task)
          pools.splice(idx)
        })
        .catch(() => {
          failChunks.push(chunk)
        })
      pools.push(task)
      if (pools.length >= maxPoolsSize) {
        // 等待并发池执行完一个任务后
        await Promise.race(pools)
      }
    }
  }
  await uploadChunks(chunks)

  // 上传失败的的
  if (failChunks.length > 0) {
    uploadChunks(failChunks)
  }
}
// 获取文件hash值
const getFileHash = (file: File): Promise<string> => {
  return new Promise((resolve, reject) => {
    const fileReader = new FileReader()
    fileReader.readAsArrayBuffer(file)
    fileReader.onload = ev => {
      let buffer = ev.target?.result
      const hash = new SparkMD5.ArrayBuffer().append(buffer! as ArrayBuffer).end()
      resolve(hash)
    }
  })
}
const reset = () => {
  uploading.value = false
}
</script>
<template>
  <section
    class="upload-box"
    @click="handleClick"
    @drop.prevent="onDrop"
    @dragover.prevent="onDragover"
  >
    <input ref="inputRef" type="file" class="hidden" @change="handleChange" @click.stop />
    <i class="icon"></i>
    <span class="text">将文件拖到此处，或<em>点击上传</em></span>
    <!-- 进度条 -->
    <div v-if="uploading" class="progress">
      <div class="progress-value" :style="{ width: `${uploadPencent}%` }">{{ uploadPencent }}%</div>
    </div>
    <div v-if="isProcessing" class="processing">正在解析文件,请稍候...</div>
  </section>
</template>
<style lang="scss" scoped>
.upload-box {
  width: 400px;
  height: 200px;
  border: 1px dashed #ccc;
  display: flex;
  flex-direction: column;
  justify-content: center;
  align-items: center;
  padding: 0 20px;
  box-sizing: border-box;
  cursor: pointer;
  .icon {
    width: 80px;
    height: 62px;
    background: url('@/assets/images/upload.png') no-repeat;
    background-size: 100% 100%;
    margin-bottom: 16px;
  }
  .text {
    font-size: 14px;
    em {
      color: skyblue;
    }
  }
  .progress {
    width: 100%;
    height: 20px;
    margin-top: 8px;
    box-sizing: border-box;
    background-color: #eee;
    border-radius: 8px;
    &-value {
      background: skyblue;
      height: 20px;
      line-height: 20px;
      text-align: right;
      padding-right: 8px;
      border-radius: 8px;
      font-size: 14px;
      color: #fff;
    }
  }
  .processing {
    width: 100%;
    height: 20px;
    margin-top: 8px;
    text-align: center;
  }
}
.hidden {
  display: none;
}
</style>