node上传文件实现断点续传

代码地址

什么是断点续传？

使用普通上传文件时，突然遇到网络断开或其他某个问题导致上传文件停止，这时重新上传文件，服务端将从头开始，小文件倒没多大问题，大文件就显得浪费资源。而断点续传就是解决这个问题，断点续传其实正如字面意思，就是在下载的断开点继续开始传输，不用再从头开始。

原理

上传文件时，可通过blob分成多个块上传，到最后的时候，将这些块合并成一个文件，在这过程中，如果文件上传停止，下次重新上传时，拿到上次上传文件的块索引，在这个索引叠加上传即可，最后上传所有再合并成一个文件。

环境

后端

formidable 文件上传模块 express Web框架

前端

axios 请求接口 spark-md5 MD5加密

创建工程

前端代码index.html

view:

<div class="upload">
    <h3>大文件上传</h3>
    <form>
        <div class="upload-file">
            <label for="file">请选择文件</label>
            <input type="file" name="file" id="big-file" accept="application/*">
        </div>
        <div class="upload-progress">
            当前进度：
            <p>
                <span style="width: 0;" id="big-current"></span>
            </p>
        </div>
    </form>
</div>

css:

        body {
            margin: 0;
            font-size: 16px;
            background: #f8f8f8;
        }
        h1,h2,h3,h4,h5,h6,p {
            margin: 0;
        }

        /* * {
            outline: 1px solid pink;
        } */

        .upload {
            box-sizing: border-box;
            margin: 30px auto;
            padding: 15px 20px;
            width: 500px;
            height: auto;
            border-radius: 15px;
            background: #fff;
        }

        .upload h3 {
            font-size: 20px;
            line-height: 2;
            text-align: center;
        }

        .upload .upload-file {
            position: relative;
            margin: 30px auto;
        }

        .upload .upload-file label {
            display: flex;
            justify-content: center;
            align-items: center;
            width: 100%;
            height: 150px;
            border: 1px dashed #ccc;
        }

        .upload .upload-file input {
            position: absolute;
            top: 0;
            left: 0;
            width: 100%;
            height: 100%;
            opacity: 0;
        }

        .upload-progress {
            display: flex;
            align-items: center;
        }

        .upload-progress p {
            position: relative;
            display: inline-block;
            flex: 1;
            height: 15px;
            border-radius: 10px;
            background: #ccc;
            overflow: hidden;
        }

        .upload-progress p span {
            position: absolute;
            left: 0;
            top: 0;
            width: 0;
            height: 100%;
            background: linear-gradient(to right bottom, rgb(163, 76, 76), rgb(231, 73, 52));
            transition: all .4s;
        }

        .upload-link {
            margin: 30px auto;
        }

        .upload-link a {
            text-decoration: none;
            color: rgb(6, 102, 192);
        }

        @media all and (max-width: 768px) {
            .upload {
                width: 300px;
            }
        }

js:

<script src="https://cdn.bootcdn.net/ajax/libs/axios/0.21.1/axios.min.js"></script>
<script src="https://cdn.bootcdn.net/ajax/libs/spark-md5/3.0.0/spark-md5.min.js"></script>
<script>
const bigFile = document.querySelector('#big-file');
    let bigCurrent = document.querySelector('#big-current');
    let bigLinks = document.querySelector('#big-links');
    let fileArr = [];
    let md5Val = '';
    let ext = '';


    bigFile.addEventListener('change', (e) => {
        let file = e.target.files[0];
        let index =   file.name.lastIndexOf('.')
        ext = file.name.substr(index + 1)
        if (file.type.indexOf('application') == -1) {
            return alert('文件格式只能是文档应用！');
        }
        if ((file.size / (1000*1000)) > 100) {
            return alert('文件不能大于100MB！');
        }
        this.uploadBig(file);
    }, false);

    // 操作上传
    async function uploadBig(file){
        let chunkIndex = 0
        fileArr  = sliceFile (file)
        md5Val =  await md5File(fileArr)
        // 获取上次上传索引
        let data =  await axios({
            url: `${baseUrl}/big?type=check&md5Val=${md5Val}&total=${fileArr.length}`,
            method: 'post',
        })
        if (data.data.code == 200) {
            chunkIndex =  data.data.data.data.chunk.length  ? data.data.data.data.chunk.length - 1 : 0
            console.log('chunkIndex', chunkIndex)
        }
        await uploadSlice(chunkIndex)
    }
    // 切割文件
    function sliceFile (file) {
        const files = [];
        const chunkSize = 128*1024;
        for (let i = 0; i < file.size; i+=chunkSize) {
            const end = i + chunkSize >= file.size ? file.size : i + chunkSize;
            let currentFile = file.slice(i, (end > file.size ? file.size : end));
            files.push(currentFile);
        }
        return files;
    }

    // 获取文件md5值
    function md5File (files) {
        const spark = new SparkMD5.ArrayBuffer();

        let fileReader;
        for (var i = 0; i < files.length; i++) {
            fileReader = new FileReader();
            fileReader.readAsArrayBuffer(files[i]);
        }
        return new Promise((resolve) => {
            fileReader.onload = function(e) {
                spark.append(e.target.result);
                if (i == files.length) {
                    resolve(spark.end());
                }
            }
        })
    }

    // 分块上传请求
    async function uploadSlice (chunkIndex = 0) {
        let formData = new FormData();
        formData.append('file', fileArr[chunkIndex]);
        let data = await axios({
            url: `${baseUrl}/big?type=upload&current=${chunkIndex}&md5Val=${md5Val}&total=${fileArr.length}`,
            method: 'post',
            data: formData,
        })

        if (data.data.code == 200) {
            if (chunkIndex < fileArr.length -1 ){
                bigCurrent.style.width = Math.round((chunkIndex+1) / fileArr.length * 100) + '%';
                ++chunkIndex;
                uploadSlice(chunkIndex);
            } else {
                mergeFile();
            }
        }
    }
    //合并文件请求
    async function mergeFile () {
        let data = await axios.post(`${baseUrl}/big?type=merge&md5Val=${md5Val}&total=${fileArr.length}&ext=${ext}`);
        if (data.data.code == 200) {
            alert('上传成功！');
            bigCurrent.style.width = '100%';
            bigLinks.href = data.data.data.url;
        } else {
            alert(data.data.data.info);
        }
    }
</script>

后端代码index.js

const express = require('express');
const formidable = require('formidable');
const path = require('path');
const fs = require('fs');
const baseUrl = 'http://localhost:3000/file/doc/';
const dirPath = path.join(__dirname, '/static/')
const  app = express()
// 解决跨域
app.all('*', function (req, res, next) {
    res.header('Access-Control-Allow-Origin', '*')
    res.header('Access-Control-Allow-Headers', 'Content-Type')

    res.header('Access-Control-Allow-Methods', '*');

    res.header('Content-Type', 'application/json;charset=utf-8')
    next();
});


app.post('/big', async function (req, res){
    let type = req.query.type;
    let md5Val = req.query.md5Val;
    let total = req.query.total;
    let bigDir = dirPath + 'big/';
    let typeArr = ['check', 'upload', 'merge'];
    if (!type) {
        return res.json({
            code: 101,
            msg: 'get_fail',
            data: {
                info: '上传类型不能为空！'
            }
        })
    }

    if (!md5Val) {
        return res.json({
            code: 101,
            msg: 'get_fail',
            data: {
                info: '文件md5值不能为空！'
            }
        })
    }

    if (!total) {
        return res.json({
            code: 101,
            msg: 'get_fail',
            data: {
                info: '文件切片数量不能为空！'
            }
        })
    }

    if (!typeArr.includes(type)) {
        return res.json({
            code: 101,
            msg: 'get_fail',
            data: {
                info: '上传类型错误！'
            }
        })
    }
   if (type === 'check') {
       let filePath = `${bigDir}${md5Val}`;
       fs.readdir(filePath, (err, data) => {
           if (err) {
               fs.mkdir(filePath, (err) => {
                   if (err) {
                       return res.json({
                           code: 101,
                           msg: 'get_fail',
                           data: {
                               info: '获取失败！',
                               err
                           }
                       })
                   } else {
                       return res.json({
                           code: 200,
                           msg: 'get_succ',
                           data: {
                               info: '获取成功！',
                               data: {
                                   type: 'write',
                                   chunk: [],
                                   total: 0
                               }
                           }
                       })
                   }
               })
           } else {
               return res.json({
                   code: 200,
                   msg: 'get_succ',
                   data: {
                       info: '获取成功！',
                       data: {
                           type: 'read',
                           chunk: data,
                           total: data.length
                       }
                   }
               })
           }

       });

   } else if (type === 'upload') {

       let current = req.query.current;
       if (!current) {
           return res.json({
               code: 101,
               msg: 'get_fail',
               data: {
                   info: '文件当前分片值不能为空！'
               }
           })
       }

       let form = formidable({
           multiples: true,
           uploadDir: `${dirPath}big/${md5Val}/`,
       })

       form.parse(req, (err,fields, files)=> {
           if (err) {
               return res.json(err);
           }
           let newPath = `${dirPath}big/${md5Val}/${current}`;
           fs.rename(files.file.path, newPath, function(err) {
               if (err) {
                   return res.json(err);
               }
               return res.json({
                   code: 200,
                   msg: 'get_succ',
                   data: {
                       info: 'upload success!'
                   }
               })
           })

       });

   } else  {
       let ext = req.query.ext;
       if (!ext) {
           return res.json({
               code: 101,
               msg: 'get_fail',
               data: {
                   info: '文件后缀不能为空！'
               }
           })
       }

       let oldPath = `${dirPath}big/${md5Val}`;
       let newPath = `${dirPath}doc/${md5Val}.${ext}`;
       let data = await mergeFile(oldPath, newPath);
       if (data.code == 200) {
           return res.json({
               code: 200,
               msg: 'get_succ',
               data: {
                   info: '文件合并成功！',
                   url: `${baseUrl}${md5Val}.${ext}`
               }
           })
       } else {
           return res.json({
               code: 101,
               msg: 'get_fail',
               data: {
                   info: '文件合并失败！',
                   err: data.data.error
               }
           })
       }
   }
})


// 多个块合并成一个文件
function mergeFile (filePath, newPath) {
    return new Promise((resolve, reject) => {
        let files = fs.readdirSync(filePath),
            newFile = fs.createWriteStream(newPath);
        let filesArr = arrSort(files).reverse();
        main();
        function main (index = 0) {
            let currentFile = filePath + '/'+filesArr[index];
            let stream = fs.createReadStream(currentFile);
            stream.pipe(newFile, {end: false});
            stream.on('end', function () {
                if (index < filesArr.length - 1) {
                    index++;
                    main(index);
                } else {
                    resolve({code: 200});
                }
            })
            stream.on('error', function (error) {
                reject({code: 102, data:{error}})
            })
        }
    })
}
// 文件排序
function arrSort (arr) {
    for (let i = 0; i < arr.length; i++) {
        for (let j = 0; j < arr.length; j++) {
            if (Number(arr[i]) >= Number(arr[j])) {
                let t = arr[i];
                arr[i] = arr[j];
                arr[j] = t;
            }
        }
    }
    return arr;
}

app.listen(3000, ()=>{
    console.log('http://localhost:3000/')
})