1.简单版

npm i http -s //安装库

//vue
<template>
  <div> 
    <input type="file" name="file" @change="handleFileChange"> 
      <el-button @click="uploadFile">上传</el-button>
  </div>
</template>

method:{
    handleFileChange(e){ 
      const [file] = e.target.files
      if(!file) return 
      this.file = file
    }
     async uploadFile(){
      if(!this.file){
        return 
      } 
      const form = new FormData() 
      form.append('file',this.file)
      form.append('name','file')
      const ret = await axios.post('/uploadfile',form) //http是封装好的axios
     }
}

//app/router.js
module.exports = app => {
  const { router, controller } = app
  router.post('/uploadfile', controller.util.uploadfile)
}

//config/config.default.js 
const path = require('path')
module.exports = appInfo => {
  const config = exports = {}
  config.UPLOAD_DIR = path.resolve(__dirname, '..', 'app/public')
}

//app/controller/util.js
const fse = require('fs-extra')
const path = require('path')
const BaseController = require('./base')

class UtilController extends BaseController { 
  async uploadfile() {
    const { ctx } = this
    console.log(ctx.request)
    const file = ctx.request.files[0]
    const { name } = ctx.request.body 
    console.log(name,file) 
    await fse.move(file.filepath, this.config.UPLOAD_DIR)

    this.message('切片上传成功')
     this.success({
       url:`/public/${file.filename}`
     })
  }
}
module.exports = UtilController

2.拖拽版

//vue
<template>
<div>
  <div ref='drag' id="drag">
    <input type="file" name="file" @change="handleFileChange"> 
      <el-button @click="uploadFile">上传</el-button>
  </div> 
   <el-progress :stroke-width='20' :text-inside="true" :percentage="uploadProgress" >
   </el-progress> 
 </div> 
</template> 

<style lang="stylus">
#drag
  height 100px
  line-height 100px
  border 2px dashed #eee
  text-align center 
</style>
   data(){
       return {
           uploadProgress : 0 //进度信息
       }
   }
   async mounted(){
     this.bindEvents()
   },
    methods:{
    bindEvents(){
      const drag = this.$refs.drag
      drag.addEventListener('dragover',e=>{//拖拽进入感应区
        drag.style.borderColor = 'red'
        e.preventDefault()//阻止冒泡
      })
      drag.addEventListener('dragleave',e=>{ //拖拽离开感应区
        drag.style.borderColor = '#eee'
        e.preventDefault()//阻止冒泡
      })
      drag.addEventListener('drop',e=>{ //拖拽停止
        const fileList = e.dataTransfer.files
        drag.style.borderColor = '#eee'
        this.file = fileList[0]
        e.preventDefault()//阻止冒泡
      })
    }, 
     async uploadFile(){
      if(!this.file){
        return 
      } 
      const form = new FormData() 
      form.append('file',this.file)
      form.append('name','file')
      //通过回调函数onUploadProgress 监听变化信息
      const ret = await axios.post('/uploadfile',form,{onUploadProgress:progress=>
           { 
            this.uploadProgress = Number(((progress.loaded/progress.total)*100).toFixed(2))
           }
         })
     }
  }

3.文件格式校验

3.1普通方式

//elementUI
//方式1
<el-upload accept="image/jpeg,image/gif,image/png" >
</el-upload>
//方式2
beforeUpload(file) {       
     var FileExt = file.name.replace(/.+\./, "");       
      if (['jpg','png','txt','zip', 'rar','pdf','doc','docx','xlsx'].indexOf(FileExt.toLowerCase()) === -1){            
        this.$message({ 
            type: 'warning', 
            message: '请上传后缀名为jpg、png、txt、pdf、doc、docx、xlsx、zip或rar的附件！' 
         });                
        return false;       
      }      
},
//大小限制 
beforeUpload(file) {     
this.isLt2k = file.size / 1024  < 200?'1':'0';        
    if(this.isLt2k==='0') {            
        this.$message({                
            message: '上传文件大小不能超过200k!',                
            type: 'error'            
        });        
    }        
return this.isLt2k==='1'?true: false;  
},

3.2进制信息方式校验

png gif gif 宽高直接获取

jpg

    methods:{ 
         async uploadFile(){
          if(!this.file){
            return 
          }
           if(!await this.isImage(this.file)){
             console.log('文件格式不对')
           }else{
             console.log('格式正确')
           }
          ....
        } 
        //二进制流转字符串  使用promise方式返回，防止io流阻塞
        async blobToString(blob){
          return new Promise(resolve=>{
            const reader = new FileReader()
            reader.onload = function(){
              console.log(reader.result)
              const ret = reader.result.split('')
                            .map(v=>v.charCodeAt()) //先把如G转10进制 71 
                            .map(v=>v.toString(16).toUpperCase()) //再把71转16进制
                            // .map(v=>v.padStart(2,'0'))
                            .join('')
              resolve(ret)
              // const ret = reader.
            }
            reader.readAsBinaryString(blob)
          })
        } ,
        async isGif(file){//头部判断
          // GIF89a 和GIF87a
          // 前面6个16进制，'47 49 46 38 39 61' '47 49 46 38 37 61'
          const ret = await this.blobToString(file.slice(0,6))
          const isGif = (ret=='47 49 46 38 39 61') || (ret=='47 49 46 38 37 61')
          return isGif
        },
        async isPng(file){//头部判断
          const ret = await this.blobToString(file.slice(0,8))
          const ispng = (ret == "89 50 4E 47 0D 0A 1A 0A")
          return ispng
        },
        async isJpg(file){//头尾判断
          const len = file.size
          const start = await this.blobToString(file.slice(0,2))
          const tail = await this.blobToString(file.slice(-2,len))
          const isjpg = (start=='FF D8') && (tail=='FF D9')
          return isjpg
        },
        async isImage(file){
          // 通过文件流来判定
          // 先判定是不是gif
          return await this.isGif(file) || await this.isPng(file)
        },
    }

4.断点续传

4.1 md5生成唯一标识(webworker)

npm i sprak-md5

//通过 file.slice(0,1024) //分割文件为多个碎片
//sprak-md5 动态计算对应的文件hash，支持分片计算md5优化md5的运行时间
// md5 解决：1.文件命名唯一问题 2.妙传功能判断条件

//vue页面

//定义文件分片方法
const CHUNK_SIZE = 10*1024*1024 //按10m分割一块
 methods:{ 
    createFileChunk(file,size=CHUNK_SIZE){
      const chunks = [] 
      let cur = 0
      while(cur<file.size){
        chunks.push({index:cur, file:file.slice(cur,cur+size)})
        cur+=size
      }
      return chunks
    },
   async uploadFile(){
      if(!this.file){
        return 
      }  
      const chunks = this.createFileChunk(this.file)  
      const hash = await this.calculateHashWorker()
   }, 
   //通过webwork动态计算hash值
    async calculateHashWorker(){
      return new Promise(resolve=>{
        this.worker = new Worker('/hash.js')
        this.worker.postMessage({chunks:this.chunks})
        this.worker.onmessage = e=>{
          const {progress,hash} = e.data
          this.hashProgress = Number(progress.toFixed(2))
          if(hash){
            resolve(hash)
          }
        }
      })
    },
}

//1.把spark-md5安装包放在static路径下
spark-md5.min.js
//2.创建hash.js
//static/hash.js
// 引入spark-md5
self.importScripts('spark-md5.min.js')
self.onmessage = e=>{
  // 接受主线程传递的数据
  const {chunks } = e.data
  const spark = new self.SparkMD5.ArrayBuffer()
  let progress = 0
  let count = 0
  const loadNext = index=>{
    const reader = new FileReader()
    reader.readAsArrayBuffer(chunks[index].file)
    reader.onload = e=>{
      count ++
      spark.append(e.target.result)

      if(count==chunks.length){
        self.postMessage({
          progress:100,
          hash:spark.end()
        })
      }else{
        progress += 100/chunks.length
        self.postMessage({
          progress
        })
        loadNext(count)
      }
    }
  }
  loadNext(0)
}

webworker交互流程 hash.js 动态解析所有分块数组逻辑

4.2 使用window.requestIdleCallback 空闲时执行md5操作

 methods:{ 
    createFileChunk(file,size=CHUNK_SIZE){
      const chunks = [] 
      let cur = 0
      while(cur<file.size){
        chunks.push({index:cur, file:file.slice(cur,cur+size)})
        cur+=size
      }
      return chunks
    },
   async uploadFile(){
      if(!this.file){
        return 
      }  
      const chunks = this.createFileChunk(this.file)  
      const hash = await this.calculateHashIdle()
   }, 
    //浏览器每一帧都有空闲，流程的动画网页 1秒 60帧，
   // 一帧大哥16.6ms 
   // |渲染，绘制16.6| 更新UI 16.6| 动画16.6| 同步任务16.6| |
    // 一般同步任务都大于16ms ，所以会阻塞，看起来不流畅
 // 60fps
    // 1秒渲染60次 渲染1次 1帧，大概16.6ms
    async calculateHashIdle(){
      const chunks = this.chunks
      return new Promise(resolve=>{
        const spark = new sparkMD5.ArrayBuffer()
        let count = 0 

        const appendToSpark = async file=>{
          return new Promise(resolve=>{
            const reader = new FileReader()
            reader.readAsArrayBuffer(file)
            reader.onload = e=>{
              spark.append(e.target.result)
              resolve()
            }
          })
        }
        const workLoop = async deadline=>{
          // timeRemaining获取当前帧的剩余时间
          while(count<chunks.length && deadline.timeRemaining()>1){ //一直while 循环操作
            // 空闲时间，且有任务
            await appendToSpark(chunks[count].file)
            count++
            if(count<chunks.length){
              this.hashProgress = Number(
                ((100*count)/chunks.length).toFixed(2)
              )
            }else{
              this.hashProgress = 100
              resolve(spark.end())
            }
          }
          window.requestIdleCallback(workLoop)
        }
        // 浏览器一旦空闲，就会调用workLoop
        window.requestIdleCallback(workLoop)

      })
    },
 }

4.3抽样方式生成MD5

通过损失一部分精度 来减少文件的大小，提升hash的速度

 methods:{  
   async uploadFile(){
      if(!this.file){
        return 
      }   
      const hash = await this.calculateHashSample()
   }, 
  async calculateHashSample(){
      // 布隆过滤器  判断一个数据存在与否
      // 1个G的文件，抽样后5M以内
      // hash一样，文件不一定一样
      // hash不一样，文件一定不一样
      return new Promise(resolve=>{
        const spark = new sparkMD5.ArrayBuffer()
        const reader = new FileReader()

        const file = this.file
        const size = file.size
        const offset = 2*1024*1024
        // 第一个2M，最后一个区块数据全要
        let chunks = [file.slice(0,offset)]//放入第一个2m的内容

        let cur = offset
        while(cur<size){
          if(cur+offset>=size){ //如果遍历到最后一个区块 加入数组
            // 最后一个区快
            chunks.push(file.slice(cur, cur+offset))

          }else{
            // 中间的区块
            const mid = cur+offset/2
            const end = cur+offset
            chunks.push(file.slice(cur, cur+2)) //当前2字节
            chunks.push(file.slice(mid, mid+2)) //当前中间2字节
            chunks.push(file.slice(end-2, end)) //尾巴2字节
          }
          cur+=offset
        }
        // 中间的，取前中后各2各字节
        reader.readAsArrayBuffer(new Blob(chunks))
        reader.onload = e=>{
          spark.append(e.target.result)
          this.hashProgress = 100
          resolve(spark.end())
        }
      })
    },

5.分片上传

<template>
  <div>
      <!-- 矩阵方形方式显示进度条 ,通过高度来显示当前已经上传的百分比 -->
      <div class="cube-container" :style="{width:cubeWidth+'px'}">
        <div class="cube" v-for="chunk in chunks" :key="chunk.name">
          <div
            :class="{
              'uploading':chunk.progress>0&&chunk.progress<100,
              'success':chunk.progress==100,
              'error':chunk.progress<0
            }"
            :style="{height:chunk.progress+'%'}"
          >
            <i class="el-icon-loading" style="color:#f56c6c" v-if="chunk.progress<100 && chunk.progress>0"></i>
          </div>
        </div>
      </div>
    </div>
</template>

<style lang="stylus">
#drag
  height 100px
  line-height 100px
  border 2px dashed #eee
  text-align center 
.cube-container
  .cube
    width 14px
    height 14px
    line-height 12px
    border  1px black solid
    background #eee
    float  left
    >.success
      background green
    >.uploading
      background blue
    >.error
      background red
</style>
<script>
method:{
     data(){
    return {
      chunks:[],
      hash:'', //记录生成的全局hash
    }
  },
  
    async uploadFile(){ 
      const chunks = this.createFileChunk(this.file)
      const hash = await this.calculateHashSample()
      this.hash = hash
      this.chunks = chunks.map((chunk,index)=>{
        // 切片的名字 hash+index
        const name = hash +'-'+ index
        return {
          hash,
          name,
          index,
          chunk:chunk.file,
          // 设置进度条，已经上传的，设为100
          progress:0
        }
      })
      await this.uploadChunks() 
      await this.mergeRequest()//上传完立即合并数据
   },
   async uploadChunks(){
      const requests = this.chunks
        .map((chunk,index)=>{
          // 转成promise
          const form = new FormData()
          form.append('chunk',chunk.chunk)
          form.append('hash',chunk.hash)
          form.append('name',chunk.name)
          // form.append('index',chunk.index)
          return {form, index:chunk.index,error:0}
        })
        .map(({form,index})=> this.$http.post('/uploadfile',form,{
          onUploadProgress:progress=>{
            // 不是整体的进度条了，而是每个区块有自己的进度条，整体的进度条需要计算
            this.chunks[index].progress = Number(((progress.loaded/progress.total)*100).toFixed(2))
          }
        }))
       await Promise.all(requests) 

    },
}，
computed:{
    /*
    *为了实现正方形效果 ， 所有长和宽的数量要一致。
    * 比如10的平方根就是3.1 ,向上取正就是 4，所以等于 4*4，16为正方形的16px宽度
    */
    cubeWidth(){
      return  Math.ceil(Math.sqrt(this.chunks.length))*16
    },
    /*
    * 总进度条 = 当前每个小块已经上传的文件大小合计 / 总文件大小
    * loaded*100为了显示百分比 %  
    */
    uploadProgress(){
      if(!this.file || this.chunks.length){
        return 0
      }
      const loaded = this.chunks.map(item=>item.chunk.size*item.progress)
                        .reduce((acc,cur)=>acc+cur,0)
      return parseInt(((loaded*100)/this.file.size).toFixed(2))
    }
  }, 
  </script>

5.1后台代码优化

//app/router.js
module.exports = app => {
  const { router, controller } = app
  router.post('/uploadfile', controller.util.uploadfile) 
  router.post('/mergefile', controller.util.mergefile)//新增合并方法
}
 

//app/controller/util.js 
class UtilController extends BaseController {  
    async uploadfile() {
        // /public/hash/(hash+index)
        // 报错
        if(Math.random()>0.3){
          return this.ctx.status = 500
        }
        const { ctx } = this
        console.log(ctx.request)
        const file = ctx.request.files[0]
        const { hash, name } = ctx.request.body
        const chunkPath = path.resolve(this.config.UPLOAD_DIR, hash)
        //判断文件名hash对应文件夹是否存在
        if (!fse.existsSync(chunkPath)) {
          await fse.mkdir(chunkPath)
        }
        await fse.move(file.filepath, `${chunkPath}/${name}`)
        this.message('切片上传成功')
    }
      //合并chunks的所有文件，还原原文件
      async mergefile() {
        const { ext, size, hash } = this.ctx.request.body
        const filePath = path.resolve(this.config.UPLOAD_DIR, `${hash}.${ext}`)
        await this.ctx.service.tools.mergeFile(filePath, hash, size)
        this.success({
          url: `/public/${hash}.${ext}`,
        })
      }
}
module.exports = UtilController


//app/service/tools.js
class ToolService extends Service {
  //遍历路径下所有文件块
  async mergeFile(filepPath, filehash, size) {
    const chunkdDir = path.resolve(this.config.UPLOAD_DIR, filehash) // 切片的文件夹
    let chunks = await fse.readdir(chunkdDir)
    //排序是为了后面合并按顺序合并，默认遍历文件夹不一定是有序
    chunks.sort((a, b) => a.split('-')[1] - b.split('-')[1])
    chunks = chunks.map(cp => path.resolve(chunkdDir, cp))
    await this.mergeChunks(chunks, filepPath, size)

  }
  //合并文件
  async mergeChunks(files, dest, size) {
    const pipStream = (filePath, writeStream) => new Promise(resolve => {
      const readStream = fse.createReadStream(filePath)
      readStream.on('end', () => {
        fse.unlinkSync(filePath) 
        resolve()
      })
      readStream.pipe(writeStream)//通过管道直接合并
    })

    await Promise.all(
      files.forEach((file, index) => {
        pipStream(file, fse.createWriteStream(dest, {
          start: index * size,
          end: (index + 1) * size,
        }))
      })
    )
  }
}

5.2上传的结果

5.2.1hash文件夹下的块文件

5.2.2合并之后

5.3.1优化loading不能立刻显示问题

提前吧loading加载好

<template> 
    <i class="el-icon-loading"></i> 
    <!-- 在下面显示的时候就能立刻显示el-icon-loading -->
    <div class="cube-container" :style="{width:cubeWidth+'px'}">
        <div class="cube" v-for="chunk in chunks" :key="chunk.name">
          <div
            :class="{
              'uploading':chunk.progress>0&&chunk.progress<100,
              'success':chunk.progress==100,
              'error':chunk.progress<0
            }"
            :style="{height:chunk.progress+'%'}"
          >
            <i class="el-icon-loading" style="color:#f56c6c" v-if="chunk.progress<100 && chunk.progress>0"></i>
          </div>
        </div>
      </div> 
</template> 

6.秒传

  async uploadFile(){
      // 问一下后端，文件是否上传过，如果没有，是否有存在的切片
      const {data:{uploaded, uploadedList}} = await this.$http.post('/checkfile',{
        hash:this.hash,
        ext:this.file.name.split('.').pop()
      })
      if(uploaded){
        // 秒传
        return this.$message.success('秒传成功')
      }
  }

//后台
//app/router.js

module.exports = app => {
  const { router, controller } = app 
  router.post('/checkfile', controller.util.checkfile)
}

//app/controller/util.js 
class UtilController extends BaseController {  
    async checkfile() {
    const { ctx } = this
    const { ext, hash } = ctx.request.body
    const filePath = path.resolve(this.config.UPLOAD_DIR, `${hash}.${ext}`)

    let uploaded = false
    let uploadedList = []
    if (fse.existsSync(filePath)) {
      // 文件存在
      uploaded = true
    } else {
      uploadedList = await this.getUploadedList(path.resolve(this.config.UPLOAD_DIR, hash))
    }
    this.success({
      uploaded,
      uploadedList,
    })
  }
  // .DS_Strore
  async getUploadedList(dirPath) {
    return fse.existsSync(dirPath)
      ? (await fse.readdir(dirPath)).filter(name => name[0] !== '.')
      : []
  }
}
module.exports = UtilController

7.断点续传

    async uploadFile(){ 
      this.chunks = chunks.map((chunk,index)=>{
        // 切片的名字 hash+index
        const name = hash +'-'+ index
        return {
          hash,
          name,
          index,
          chunk:chunk.file,
          // 设置进度条，已经上传的，设为100
           progress:uploadedList.indexOf(name)>-1 ?100:0 
        }
      }) 
      await this.uploadChunks(uploadedList)
   }, 
    async uploadChunks(uploadedList=[]){//新增参数uploadedList
      const requests = this.chunks
        .filter(chunk=>uploadedList.indexOf(chunk.name)==-1) //添加过滤。如果已经上传的不加入promies中
        .map((chunk,index)=>{
          // 转成promise
          const form = new FormData()
          form.append('chunk',chunk.chunk)
          form.append('hash',chunk.hash)
          form.append('name',chunk.name)
          // form.append('index',chunk.index)
          return {form, index:chunk.index,error:0}
        })
    },

8.并发数控制

不控制的问题

1. 过多的请求等待，会导致浏览器卡顿
2. 并且会阻塞后面新的请求
3. 虽然浏览器会限制并发的数量，但是padding的请求会一直占用资源

      //await Promise.all(requests) 
      await this.sendRequest(requests)//改用并发控制 

//统一处理并发的请求
async sendRequest(chunks,limit=4){
      // limit仕并发数
      // 一个数组,长度仕limit  通过头部挤出，尾部加入，先进先出的方式
      // [task12,task13,task4]
      return new Promise((resolve,reject)=>{
        const len = chunks.length
        let counter = 0  
        const start = async ()=>{ 
          const task = chunks.shift()
          if(task){
            const {form,index} = task 
              await this.$http.post('/uploadfile',form,{
                onUploadProgress:progress=>{
                  // 不是整体的进度条了，而是每个区块有自己的进度条，整体的进度条需要计算
                  this.chunks[index].progress = Number(((progress.loaded/progress.total)*100).toFixed(2))
                }
              })
              if(counter==len-1){
                // 最后一个任务
                resolve()
              }else{
                counter++
                // 启动下一个任务
                start()
              } 
          }
        } 
        while(limit>0){
          // 启动limit个任务
          // 模拟一下延迟 ，因为每个文件大小都差不多，所以通过限制启动时间，默认先后顺序的效果
          setTimeout(()=>{
            start()
          },Math.random()*2000)
          limit-=1
        } 
      })
    },

8.1 并发报错处理

//后端模拟报错
class UtilController extends BaseController {
    async uploadfile() {
        // /public/hash/(hash+index)
        // 报错
        if(Math.random()>0.3){
          return this.ctx.status = 500
        }
        const { ctx } = this
    }
}

    //前端
    //上传可能报错
    // 报错之后，进度条变红，开始重试
    // 在task里面新增task.error 属性，一个切片重试失败三次，整体全部终止 isStop = true
    async sendRequest(chunks,limit=4){
      // limit仕并发数
      // 一个数组,长度仕limit
      // [task12,task13,task4]
      return new Promise((resolve,reject)=>{
        const len = chunks.length
        let counter = 0 
        let isStop = false
        const start = async ()=>{
          if(isStop){
            return 
          }
          const task = chunks.shift()
          if(task){
            const {form,index} = task

            try{
              await this.$http.post('/uploadfile',form,{
                onUploadProgress:progress=>{
                  // 不是整体的进度条了，而是每个区块有自己的进度条，整体的进度条需要计算
                  this.chunks[index].progress = Number(((progress.loaded/progress.total)*100).toFixed(2))
                }
              })
              if(counter==len-1){
                // 最后一个任务
                resolve()
              }else{
                counter++
                // 启动下一个任务
                start()
              }
            }catch(e){

              this.chunks[index].progress = -1
              if(task.error<3){
                task.error++
                chunks.unshift(task)//这里立刻加入头部，在start()的时候会优先执行
                start()
              }else{
                // 错误三次
                isStop = true
                reject() //promise 对错都应该要返回对应事件
              }
            }
          }
        }

        while(limit>0){
          // 启动limit个任务
          // 模拟一下延迟
          setTimeout(()=>{
            start()
          },Math.random()*2000)
          limit-=1
        }
      
      })
    },

8.2扩展知识，如何实现慢启动

TCP慢启动，先上传一个初始区块，比如10KB，根据上传成功时间，决定下一个区块仕20K，hi是50K，还是5K 在下一个一样的逻辑，可能编程100K，200K，或者2K