基于nuxt实现大文件上传

前言

最近在学习大文件上传相关的知识，把自己学到的知识总结一下。

主要的知识点：

• 通过二进制信息确认文件格式
• 三种方式计算文件hash
• 切片上传
• 秒传和断点续传
• 并发控制和报错重试

1、项目搭建

1.1 运行命令，初始化项目

npx create-nuxt-app nuxt-demo

安装完后，进入项目目录cd nuxt-demo

1.2 安装项目所需要的依赖

npm i stylus stylus-loader@3.0.2 koa-body koa-router fs-extra spark-md5

stylus 用于样式处理

koa-body 用于获取表单的body数据

koa-router 用于提供路由

fs-extra 是fs的扩展，提供了很多方便的文件操作方法

spark-md5用于计算md5

1.3 axios配置

在plugins文件夹里面创建axios.js

// plugins/axios.js
import Vue from 'vue'
import axios from 'axios'
let service = axios.create({
  // 前缀
  baseURL:'/api'
})

Vue.prototype.$http =service

export const http = service

在nuxt.config.js配置axios 插件

...
plugins: [
    '@/plugins/element-ui',
    '@/plugins/axios'
],

1.4 配置koa

在server文件夹新增upload.js， 用于提示文件上传的接口

// server/upload.js
const KoaRouter = require('koa-router')

const router = new KoaRouter({
    prefix: '/api'
})

router.get('/hello', ctx => {
    ctx.body = 'hello koa'
})

module.exports = router

在server/index.js 引入 koa-body 和upload.js

...
const { Nuxt, Builder } = require('nuxt')
const KoaBody = require('koa-body')
const uploadInterface = require('./upload')


app.use(KoaBody({ multipart: true }));
app.use(uploadInterface.routes()).use(uploadInterface.allowedMethods())

app.use((ctx) => {
    ctx.status = 200
    ctx.respond = false // Bypass Koa's built-in response handling
    ctx.req.ctx = ctx // This might be useful later on, e.g. in nuxtServerInit or with nuxt-stash
    nuxt.render(ctx.req, ctx.res)
})

启动项目

npm run dev

这时，访问 http://localhost:3000/api/hello ，如果显示hello koa 则表示koa 配置成功。

2. 实现简单的文件上传

在page新建upload.vue

// page/upload.vue
<template>
  <div>
      <div  id="drag">
        <input type="file" name="file" @change="handleFileChange" />
      </div>
      <div>
        <el-progress
          :stroke-width="20"
          :text-inside="true"
          :percentage="uploadProgress"
        ></el-progress>
      </div>
      <div>
        <el-button @click="uploadFile">上传</el-button>
      </div>
  </div>
</template>

<script>
export default {
  data(){
    return{
      uploadProgress:0
    }
  },
  methods:{
    handleFileChange(e){
      this.file = e.target.files[0]
    },
    async uploadFile(){
      if(!this.file){return}
      // 创建formData
      const formData =  new FormData()
      formData.append('file', this.file)
      formData.append('name', this.file.name)
      //发起请求
      const ret = await this.$http.post('/uploadfile', formData)
      console.log(ret);
    }
  }
}
</script>

<style lang="stylus">
#drag
  height 100px
  line-height 100px
  border 2px dashed #eee
  text-align center

</style>

在server/upload.js 新增一个uploadfile路由

const KoaRouter = require('koa-router')
const path = require('path')
const fse = require('fs-extra')
const UPLOAD_DIR = path.resolve(__dirname, '../static/upload')

const router = new KoaRouter({
    prefix: '/api'
})

router.get('/hello', ctx => {
    ctx.body = 'hello koa'
})

router.post('/uploadfile', ctx => {
    const { name } = ctx.request.body
    const file = ctx.request.files.file
    // 判断文件是否存在，没有则把临时文件拷贝到文件上传的目录
    const dest = path.resolve(UPLOAD_DIR, name)
    if (!fse.existsSync(dest)) {
        fse.moveSync(file.path, dest)
        ctx.body = { url: `/upload/${name}`, message: '文件上传成功' }
    } else {
        ctx.body = { message: "文件已经存在" }
    }
})

module.exports = router

逻辑比较简单，其实就是通过koa-body获取到请求里面的name和file字段，然后把文件从临时目录拷贝到上传的目录里面。

3. 实现拖拽和进度条

给div添加一个drop事件即可获取到拖拽的文件信息

mounted() {
    this.bindEvent();
},
methods: {
    bindEvent() {
        const dragEle = this.$refs.drag;
        dragEle.addEventListener("dragover", (e) => {
            drag.style.borderColor = "red";
            e.preventDefault();
        });
        dragEle.addEventListener("dragleave", (e) => {
            drag.style.borderColor = "#eee";
            e.preventDefault();
        });
        dragEle.addEventListener("drop", (e) => {
            const fileList = e.dataTransfer.files;
            drag.style.borderColor = "#eee";
            this.file = fileList[0];
            console.log(this.file);
            e.preventDefault();
        });
    },
}

进度条则只要配置axios里面的onUploadProgress选项即可。

const ret = await this.$http.post("/uploadfile", formData, {
    onUploadProgress: (progress) => {
        console.log(progress);
        this.uploadProgress = ((progress.loaded / progress.total) * 100) | 0;
    },
});

4. 二进制信息确认文件格式

确定一个文件是否是图片，可以通过它的二进制信息来判断

文件头信息：

• gif GIF89a '47 49 46 38 39 61' GIF87a '47 49 46 38 37 61'
• jpeg 前两位 FF D8 后两位FF D9
• png 前八位 89 50 4E 47 0D 0A 1A 0A

// upload.vue

async blobToStr(blob) {
  return new Promise((resolve) => {
    const fileReader = new FileReader();
    fileReader.onload = () => {
      const str = fileReader.result
        .split("")
        .map((v) => v.charCodeAt())  //转成字符编码
        .map((v) => v.toString(16).toUpperCase())  // 转成16进制并大写
        .map((v) => (v.length < 2 ? "0" + v : v))  // 不足两位的前面补0
        .join(" ");
      resolve(str);
    };
    fileReader.readAsBinaryString(blob);
  });
},
async isGif(file) {
  // 判断文件二进制前六位信息
  const ret = await this.blobToStr(file.slice(0, 6));
  console.log(ret);
  return ret === "47 49 46 38 39 61" || ret === "47 49 46 38 37 61";
},
async isPng(file) {
  // 前八位 '89 50 4E 47 0D 0A 1A 0A'
  const ret = await this.blobToStr(file.slice(0, 8));
  return ret === "89 50 4E 47 0D 0A 1A 0A";
},
async isJpg(file) {
  // 前两位 'FF D8' 后两位'FF D9'
  const head = await this.blobToStr(file.slice(0, 2));
  const tail = await this.blobToStr(file.slice(-2));
  return head === "FF D8" && tail === "FF D9";
},
async isImage(file) {
  return (
    (await this.isGif(file)) ||
    (await this.isPng(file)) ||
    (await this.isJpg(file))
  );
},
async uploadFile() {
  if (!this.file) {
    return;
  }

  // 判断图片格式
  if (!(await this.isImage(this.file))) {
    this.$message.warning("请选择图片");
    return;
  }
  const formData = new FormData();
  formData.append("file", this.file);
  formData.append("name", this.file.name);
  //发起请求
  const ret = await this.$http.post("/uploadfile", formData, {
    onUploadProgress: (progress) => {
      this.uploadProgress = ((progress.loaded / progress.total) * 100) | 0;
    },
  });
  console.log(ret);
},

blobToStr 的逻辑主要是通过FileReader把文件读取成字符串，然后再把字符串转成ascii编码再转成16进制。

5. 三种方式计算文件md5

先把文件切分成块

async createFileChunks(file) {
    const chunks = [];
    let cur = 0;
    while (cur < file.size) {
        chunks.push({ index: cur, chunk: file.slice(cur, cur + CHUNK_SIZE) });
        cur += CHUNK_SIZE;
    }	
    return chunks;
},

安装依赖 spark-md5，这个插件可以追加文件，最后再算出md5

5.1、web worker

把node_modules/spark-md5/spark-md5.min.js拷贝到static目录里面

在static新建hash.js文件

// 引入spark
self.importScripts("./spark-md5.min.js");

self.onmessage = e => {
    const { chunks } = e.data;
    const spark = new SparkMD5.ArrayBuffer();
    let len = chunks.length;

    const loadNext = cur => {
        const fileReader = new FileReader();
        fileReader.readAsArrayBuffer(chunks[cur].chunk);
        fileReader.onload = () => {
            spark.append(fileReader.result);
            cur++;
            if (cur < len) {
                const progress = Number(((cur * 100) / chunks.length).toFixed(2));
                self.postMessage({ progress });
                loadNext(cur);
            } else {
                self.postMessage({ progress: 100, hash: spark.end() });
            }
        };
    };

    loadNext(0);
};

新建一个方法calculateHashWorker

async calculateHashWorker(chunks) {
    return new Promise(resolve => {
        const worker = new Worker("hash.js");
        worker.postMessage({ chunks });
        worker.onmessage = e => {
            const { progress, hash } = e.data;
            if (!hash) {
                this.hashProgress = progress;
            } else {
                this.hashProgress = 100;
                resolve(hash);
            }
        };
    });
}

思路：先创建一个worker，然后通过postMessage把chunks传给worker，worker接收到数据后，通过loadNext方法，一次只处理一个chunk，处理完后就把进度回传给主进程，直到所有chunk处理完后把hash返回来。

5.2、requestIdleCallback

再通过spark-md5对chunks进行md5计算

async calculateHashIdle(chunks) {
    return new Promise(resolve => {
        let count = 0;
        const spark = new sparkMD5.ArrayBuffer();
        const appendToSpark = file => {
            return new Promise(resolve => {
                const fileReader = new FileReader();
                fileReader.onload = () => {
                    spark.append(fileReader.result);
                    resolve();
                };
                fileReader.readAsArrayBuffer(file);
            });
        };
        const workLoop = async deadling => {
            while (count < chunks.length && deadling.timeRemaining() > 1) {
                // 把chunk加入spark
                await appendToSpark(chunks[count].chunk);
                count++;
                if (count < chunks.length) {
                    this.hashProgress = Number(
                        ((count * 100) / chunks.length).toFixed(2)
                    );
                } else {
                    this.hashProgress = 100;
                    resolve(spark.end());
                }
            }
            window.requestIdleCallback(workLoop);
        };

        window.requestIdleCallback(workLoop);
    });
},

5.3、抽样hash计算

async calculateHashSample(chunks) {
    return new Promise(resolve => {
        let tempChunks = [];
        // 截取chunks
        for (let i = 0; i < chunks.length; i++) {
            let chunk = chunks[i].chunk;
            if (i === 0 || i === chunks.length - 1) {
                tempChunks.push(chunk);
            } else {
                const mid = CHUNK_SIZE >> 1;
                tempChunks.push(chunk.slice(0, 2));
                tempChunks.push(chunk.slice(mid, mid + 2));
                tempChunks.push(chunk.slice(-2));
            }
        }
        // 通过fileReader把blob加入到spark
        const fileReader = new FileReader();
        const spark = new sparkMD5.ArrayBuffer();
        fileReader.readAsArrayBuffer(new Blob(tempChunks));
        fileReader.onload = () => {
            spark.append(fileReader.result);
            resolve(spark.end());
        };
    });
},

思路：切片时只切第一块和最后一块，中间的块只取前中后两个字节

6.实现切片上传合并

前端实现

// upload.vue
<template>
...
<div class="cube-container" :style="{ width: cubeWidth + 'px' }">
      <div class="cube" v-for="chunk in chunks" :key="chunk.name">
        <div
          :class="{
            uploading: chunk.progress > 0 && chunk.progress < 100,
            success: chunk.progress === 100,
            error: chunk.progress === -1,
          }"
          :style="{ height: chunk.progress + '%' }"
        >
          <i
            class="el-icon-loading"
            style="color: #f56c6c"
            v-if="chunk.progress < 100 && chunk.progress > 0"
          ></i>
        </div>
      </div>
    </div>
</template>
<script>
export default {
  data() {
    return {
      uploadProgress: 0,
      chunks: [],
    };
  },
  mounted() {
    this.bindEvent();
  },
  computed: {
    cubeWidth() {
      return Math.ceil(Math.sqrt(this.chunks.length)) * 16;
    },
    ext() {
      return this.file.name.split(".")[1];
    },
  },
   methods:{
       ...
       async uploadFile() {
      if (!this.file) {
        return;
      }

      // 判断图片格式
      if (!(await this.isImage(this.file))) {
        this.$message.warning("请选择图片");
        return;
      }
      // 切片
      const chunks = await this.createFileChunks(this.file);
      // 计算hash
      const hash = await this.calculateHashWorker(chunks);
      //   const hash2 = await this.calculateHashIdle(chunks);
      //   const hash3 = await this.calculateHashSample(chunks);
      this.hash = hash;
      this.chunks = chunks.map((chunk, index) => {
        const name = `${hash}_${index}`;
        return {
          name,
          hash,
          index,
          chunk: chunk.chunk,
          progress: 0,
        };
      });
      await this.uploadChunks();
      await this.mergeRequest();
    },
    async uploadChunks() {
      return new Promise((resolve) => {
        const chunks = this.chunks;
        const requests = chunks
          .map((chunk) => {
            // 构建表单数据
            const formData = new FormData();
            formData.append("chunk", chunk.chunk);
            formData.append("name", chunk.name);
            formData.append("hash", chunk.hash);
            return { formData, chunk };
          })
          .map(({ formData, chunk }) => {
            // 发起请求
            return this.$http.post("/uploadChunk", formData, {
              onUploadProgress: (progressEvent) => {
                let complete =
                  ((progressEvent.loaded / progressEvent.total) * 100) | 0;
                chunk.progress = complete;
              },
            });
          });
        Promise.all(requests).then(() => {
          resolve();
        });
      });
    },
    async mergeRequest() {
      await this.$http.post("/mergefile", {
        hash: this.hash,
        ext: this.ext,
        size: CHUNK_SIZE,
      });
   }
    
</script>
<style >
 ...
.cube-container {
  .cube {
    width: 14px;
    height: 14px;
    line-height: 12px;
    border: 1px black solid;
    background: #eee;
    float: left;

    >.success {
      background: green;
    }

    >.uploading {
      background: blue;
    }

    >.error {
      background: red;
    }
  }
}
</style>

在template增加切块上传的进度, 增加uploadChunks方法，用于批量上传chunks，当所有chunks上传成功后，再通过mergeRequest方法合并所有chunks。

后端实现

后端需要提供两个方法，一个用于上传chunks，一个用于合并chunks

// server/upload.js
...
router.post("/uploadChunk", async ctx => {
    const { hash, name } = ctx.request.body;
    const file = ctx.request.files.chunk;
    const uploadDir = path.resolve(UPLOAD_DIR, hash);
    if (!fse.existsSync(uploadDir)) {
        fse.mkdirSync(uploadDir);
    }
    fse.moveSync(file.path, path.resolve(uploadDir, name));
    ctx.body = { code: 0, message: "上传成功" };
});

router.post("/mergefile", async ctx => {
    const { hash, ext } = ctx.request.body;
    const filename = path.resolve(UPLOAD_DIR, `${hash}.${ext}`);
    const dirname = path.resolve(UPLOAD_DIR, hash);
    if (!fse.existsSync(dirname)) {
        ctx.body = { code: 0, message: "文件目录不存在" };
        return;
    }
    const uploadList = fse
        .readdirSync(dirname)
        .map(v => path.resolve(dirname, v))
        .sort((a, b) => a.split("_")[1] - b.split("_")[1]);
    for (let i = 0; i < uploadList.length; i++) {
        fse.appendFileSync(filename, fse.readFileSync(uploadList[i]));
        fse.unlink(uploadList[i]);
    }
    ctx.body = { code: 0, message: "合并成功" };
});

7. 秒传和断点续传

秒传的实现逻辑：前端向后端发起请求，询问文件是否存在，存在则提示秒传成功。

断点续传的实现逻辑：前端向后端发起请求，询问文件是否存在或者是否已经上传过，后端返回上传过的片段，前端在上传先把已经上传的过滤掉，再进行上传。

这样，后端需要提供一个checkfile方法用于判断文件是否上传，返回uploaded和uploadList。

//server/upload.js
...
router.get("/checkfile", async ctx => {
    const { hash, ext } = ctx.request.query;
    const filePath = path.resolve(UPLOAD_DIR, `${hash}.${ext}`);
    let uploaded = false;
    let uploadList = [];
    if (fse.existsSync(filePath)) {
        uploaded = true;
    } else {
        //   读取目录
        uploadList = getUploadList(path.resolve(UPLOAD_DIR, hash));
    }
    ctx.body = { uploaded, uploadList };
});

function getUploadList(dirPath) {
    return fse.existsSync(dirPath) ? fse.readdirSync(dirPath) : [];
}

前端

// upload.vue
async uploadFile() {
	...
      // 切片
      const chunks = await this.createFileChunks(this.file);
      // 计算hash
      const hash = await this.calculateHashWorker(chunks);
      //   const hash2 = await this.calculateHashIdle(chunks);
      //   const hash3 = await this.calculateHashSample(chunks);
      // 检查文件是否已经上传
      const { uploaded, uploadList } = await this.checkFile();
      if (uploaded) {
        this.$message.success("秒传成功");
        return;
      }
      ...
      await this.uploadChunks(uploadList);
      await this.mergeRequest();
},
async checkFile() {
    const ret = await this.$http.get(
        `/checkfile?hash=${this.hash}&ext=${this.ext}`
    );
    console.log(ret);
    return ret.data;
},
async uploadChunks(uploadList) {
    return new Promise((resolve) => {
        const chunks = this.chunks;
        const requests = chunks
        // 把上传过的chunk进度设置为100
        .map((chunk) => {
            if (uploadList.includes(chunk.name)) {
                chunk.progress = 100;
            }
            return chunk;
        })
        // 过滤掉上传过的chunk
        .filter((chunk) => chunk.progress !== 100)
        .map((chunk) => {
            // 构建表单数据
            const formData = new FormData();
            formData.append("chunk", chunk.chunk);
            formData.append("name", chunk.name);
            formData.append("hash", chunk.hash);
            return { formData, chunk };
        })
        ...
}

8.并发控制和报错重试

async uploadChunks(uploadList) {
    const chunks = this.chunks;
    // return new Promise(resolve => {
    const requests = chunks
    .map((chunk) => {
        if (uploadList.includes(chunk.name)) {
            chunk.progress = 100;
        }
        return chunk;
    })
    .filter((chunk) => chunk.progress !== 100)
    .map((chunk) => {
        const formData = new FormData();
        formData.append("chunk", chunk.chunk);
        formData.append("name", chunk.name);
        formData.append("hash", chunk.hash);
        return { formData, chunk };
    });
    await this.sendRequests(requests);
},
async sendRequests(tasks, limit = 4) {
    return new Promise((resolve, reject) => {
        let isStop = false;
        const len = tasks.length;
        let count = 0;
        const next = async () => {
            if (isStop) {
                return;
            }
            const task = tasks.shift();
            if (task) {
                task.error = task.error || 0;
                const { chunk, formData } = task;
                try {
                    await this.$http.post("/uploadChunk", formData, {
                        onUploadProgress: (progressEvent) => {
                            let complete =
                                ((progressEvent.loaded / progressEvent.total) * 100) | 0;
                            chunk.progress = complete;
                        },
                    });
                    count++;
                    if (count < len) {
                        next();
                    } else {
                        resolve();
                    }
                } catch (e) {
                    // 显示错误
                    chunk.progress = -1;
                    if (task.error < 3) {
                        task.error++;
                        tasks.unshift(task);
                        next();
                    } else {
                        isStop = true;
                        reject();
                    }
                }
            }
        };
        while (limit > 0) {
            next();
            limit--;
        }
    });
},

并发控制原理：通过while循环来控制同时发起请求的个数，当请求回来后接着发起下一个请求。

报错重试原理：当某个请求出现错误时，把这个请求的错误数加1，并把该任务再加入到任务队列中，如果超过3次出错，则把isStop设为true结束所有请求。

github 地址

最后附上github地址 ，有兴趣的同学可以下载使用或者研究研究，demo有问题或者写的不好改进的地方也可以互相探讨下。如果有收获的话欢迎给个start，有意见也可以随时留言反馈